2024-11-22 23:54:51,510 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-22 23:54:51,530 main DEBUG Took 0.018356 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:54:51,531 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:54:51,531 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:54:51,532 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:54:51,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,542 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:54:51,557 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,560 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,561 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,562 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,563 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,564 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,566 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,567 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,568 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,569 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,570 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:54:51,571 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,571 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:54:51,573 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:54:51,574 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:54:51,576 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:54:51,577 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:54:51,578 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:54:51,578 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:54:51,589 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:54:51,592 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:54:51,594 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:54:51,595 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:54:51,595 main DEBUG createAppenders(={Console}) 2024-11-22 23:54:51,596 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-22 23:54:51,596 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-22 23:54:51,597 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-22 23:54:51,597 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:54:51,598 main DEBUG OutputStream closed 2024-11-22 23:54:51,598 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:54:51,598 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:54:51,599 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-22 23:54:51,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:54:51,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:54:51,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:54:51,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:54:51,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:54:51,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:54:51,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:54:51,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:54:51,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:54:51,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:54:51,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:54:51,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:54:51,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:54:51,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:54:51,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:54:51,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:54:51,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:54:51,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:54:51,695 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:54:51,695 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-22 23:54:51,695 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:54:51,696 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-22T23:54:51,711 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-22 23:54:51,714 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:54:51,715 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:54:51,775 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=9366 2024-11-22T23:54:52,079 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c 2024-11-22T23:54:52,714 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:54:52,821 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:54:52,884 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-22T23:54:52,960 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319692945 2024-11-22T23:54:53,050 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319692945 2024-11-22T23:54:53,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,155 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-22T23:54:53,155 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-22T23:54:53,196 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4e1f1650362b6e324445573467dc3bec, NAME => 'testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c 2024-11-22T23:54:53,252 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:54:53,257 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:54:53,349 INFO [StoreOpener-4e1f1650362b6e324445573467dc3bec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region 4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,387 INFO [StoreOpener-4e1f1650362b6e324445573467dc3bec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e1f1650362b6e324445573467dc3bec columnFamilyName MyCF 2024-11-22T23:54:53,394 DEBUG [StoreOpener-4e1f1650362b6e324445573467dc3bec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:54:53,400 INFO [StoreOpener-4e1f1650362b6e324445573467dc3bec-1 {}] regionserver.HStore(327): Store=4e1f1650362b6e324445573467dc3bec/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:54:53,403 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/data/default/testLockupAroundBadAssignSync/4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/data/default/testLockupAroundBadAssignSync/4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,414 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,415 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,422 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e1f1650362b6e324445573467dc3bec 2024-11-22T23:54:53,434 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/data/default/testLockupAroundBadAssignSync/4e1f1650362b6e324445573467dc3bec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:54:53,436 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e1f1650362b6e324445573467dc3bec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59314390, jitterRate=-0.11614671349525452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:54:53,444 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:54:53,459 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e1f1650362b6e324445573467dc3bec: Writing region info on filesystem at 1732319693313Initializing all the Stores at 1732319693314 (+1 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732319693314Cleaning up temporary data from old regions at 1732319693415 (+101 ms)Region opened successfully at 1732319693456 (+41 ms) 2024-11-22T23:54:53,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319692945) roll requested 2024-11-22T23:54:53,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693518 2024-11-22T23:54:53,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319692945 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693518 2024-11-22T23:54:53,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693518) roll requested 2024-11-22T23:54:53,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693535 2024-11-22T23:54:53,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693535 2024-11-22T23:54:53,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693535) roll requested 2024-11-22T23:54:53,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693561 2024-11-22T23:54:53,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693561 2024-11-22T23:54:53,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693561) roll requested 2024-11-22T23:54:53,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693575 2024-11-22T23:54:53,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693575 2024-11-22T23:54:53,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693575) roll requested 2024-11-22T23:54:53,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693593 2024-11-22T23:54:53,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693593 2024-11-22T23:54:53,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693593) roll requested 2024-11-22T23:54:53,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693606 2024-11-22T23:54:53,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693606 2024-11-22T23:54:53,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693606) roll requested 2024-11-22T23:54:53,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693626 2024-11-22T23:54:53,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693626 2024-11-22T23:54:53,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693626) roll requested 2024-11-22T23:54:53,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693641 2024-11-22T23:54:53,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693641 2024-11-22T23:54:53,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693641) roll requested 2024-11-22T23:54:53,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693654 2024-11-22T23:54:53,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693654 2024-11-22T23:54:53,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693654) roll requested 2024-11-22T23:54:53,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693666 2024-11-22T23:54:53,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693666 2024-11-22T23:54:53,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693666) roll requested 2024-11-22T23:54:53,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693680 2024-11-22T23:54:53,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693680 2024-11-22T23:54:53,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693666 is not closed yet, will try archiving it next time 2024-11-22T23:54:53,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693680) roll requested 2024-11-22T23:54:53,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693695 2024-11-22T23:54:53,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693695 2024-11-22T23:54:53,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693695) roll requested 2024-11-22T23:54:53,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693708 2024-11-22T23:54:53,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693708 2024-11-22T23:54:53,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693708) roll requested 2024-11-22T23:54:53,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693723 2024-11-22T23:54:53,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693723 2024-11-22T23:54:53,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693723) roll requested 2024-11-22T23:54:53,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693734 2024-11-22T23:54:53,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693734 2024-11-22T23:54:53,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693734) roll requested 2024-11-22T23:54:53,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693746 2024-11-22T23:54:53,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693746 2024-11-22T23:54:53,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693746) roll requested 2024-11-22T23:54:53,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693757 2024-11-22T23:54:53,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693757 2024-11-22T23:54:53,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693757) roll requested 2024-11-22T23:54:53,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693767 2024-11-22T23:54:53,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693767 2024-11-22T23:54:53,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693767) roll requested 2024-11-22T23:54:53,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693777 2024-11-22T23:54:53,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693777 2024-11-22T23:54:53,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693777) roll requested 2024-11-22T23:54:53,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693787 2024-11-22T23:54:53,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693787 2024-11-22T23:54:53,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693787) roll requested 2024-11-22T23:54:53,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693797 2024-11-22T23:54:53,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693797 2024-11-22T23:54:53,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693797) roll requested 2024-11-22T23:54:53,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693816 2024-11-22T23:54:53,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693816 2024-11-22T23:54:53,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693816) roll requested 2024-11-22T23:54:53,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693827 2024-11-22T23:54:53,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693827 2024-11-22T23:54:53,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693827) roll requested 2024-11-22T23:54:53,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693843 2024-11-22T23:54:53,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693843 2024-11-22T23:54:53,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693843) roll requested 2024-11-22T23:54:53,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693856 2024-11-22T23:54:53,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693856 2024-11-22T23:54:53,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693843 is not closed yet, will try archiving it next time 2024-11-22T23:54:53,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693856) roll requested 2024-11-22T23:54:53,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693865 2024-11-22T23:54:53,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693865 2024-11-22T23:54:53,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693865) roll requested 2024-11-22T23:54:53,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693876 2024-11-22T23:54:53,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693876 2024-11-22T23:54:53,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693876) roll requested 2024-11-22T23:54:53,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693886 2024-11-22T23:54:53,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693886 2024-11-22T23:54:53,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693886) roll requested 2024-11-22T23:54:53,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693908 2024-11-22T23:54:53,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693908 2024-11-22T23:54:53,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693908) roll requested 2024-11-22T23:54:53,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693917 2024-11-22T23:54:53,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693917 2024-11-22T23:54:53,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693917) roll requested 2024-11-22T23:54:53,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693932 2024-11-22T23:54:53,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693932 2024-11-22T23:54:53,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693932) roll requested 2024-11-22T23:54:53,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693949 2024-11-22T23:54:53,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693949 2024-11-22T23:54:53,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693949) roll requested 2024-11-22T23:54:53,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693960 2024-11-22T23:54:53,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693960 2024-11-22T23:54:53,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693960) roll requested 2024-11-22T23:54:53,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693969 2024-11-22T23:54:53,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693969 2024-11-22T23:54:53,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693969) roll requested 2024-11-22T23:54:53,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693978 2024-11-22T23:54:53,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693978 2024-11-22T23:54:53,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693978) roll requested 2024-11-22T23:54:53,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693987 2024-11-22T23:54:53,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:53,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693987 2024-11-22T23:54:53,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:53,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:53,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693987) roll requested 2024-11-22T23:54:53,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319693995 2024-11-22T23:54:54,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693995 2024-11-22T23:54:54,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319693995) roll requested 2024-11-22T23:54:54,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694004 2024-11-22T23:54:54,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319693995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694004 2024-11-22T23:54:54,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694004) roll requested 2024-11-22T23:54:54,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694014 2024-11-22T23:54:54,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694014 2024-11-22T23:54:54,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694014) roll requested 2024-11-22T23:54:54,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694028 2024-11-22T23:54:54,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694028 2024-11-22T23:54:54,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694028) roll requested 2024-11-22T23:54:54,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694037 2024-11-22T23:54:54,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694037 2024-11-22T23:54:54,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694037) roll requested 2024-11-22T23:54:54,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694047 2024-11-22T23:54:54,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694047 2024-11-22T23:54:54,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694047) roll requested 2024-11-22T23:54:54,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694061 2024-11-22T23:54:54,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694061 2024-11-22T23:54:54,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694061) roll requested 2024-11-22T23:54:54,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694077 2024-11-22T23:54:54,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694077 2024-11-22T23:54:54,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694077) roll requested 2024-11-22T23:54:54,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694086 2024-11-22T23:54:54,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694086 2024-11-22T23:54:54,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694086) roll requested 2024-11-22T23:54:54,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694095 2024-11-22T23:54:54,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694095 2024-11-22T23:54:54,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694095) roll requested 2024-11-22T23:54:54,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694104 2024-11-22T23:54:54,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694104 2024-11-22T23:54:54,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694104) roll requested 2024-11-22T23:54:54,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694113 2024-11-22T23:54:54,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694113 2024-11-22T23:54:54,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694113) roll requested 2024-11-22T23:54:54,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694125 2024-11-22T23:54:54,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694125 2024-11-22T23:54:54,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694125) roll requested 2024-11-22T23:54:54,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694138 2024-11-22T23:54:54,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694138 2024-11-22T23:54:54,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694138) roll requested 2024-11-22T23:54:54,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694147 2024-11-22T23:54:54,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694147 2024-11-22T23:54:54,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694147) roll requested 2024-11-22T23:54:54,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694157 2024-11-22T23:54:54,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694157 2024-11-22T23:54:54,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694157) roll requested 2024-11-22T23:54:54,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694177 2024-11-22T23:54:54,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694177 2024-11-22T23:54:54,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694177) roll requested 2024-11-22T23:54:54,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694187 2024-11-22T23:54:54,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694187 2024-11-22T23:54:54,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694187) roll requested 2024-11-22T23:54:54,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694197 2024-11-22T23:54:54,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694197 2024-11-22T23:54:54,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694197) roll requested 2024-11-22T23:54:54,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694206 2024-11-22T23:54:54,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694206 2024-11-22T23:54:54,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694206) roll requested 2024-11-22T23:54:54,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694215 2024-11-22T23:54:54,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694215 2024-11-22T23:54:54,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694215) roll requested 2024-11-22T23:54:54,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694225 2024-11-22T23:54:54,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694225 2024-11-22T23:54:54,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694225) roll requested 2024-11-22T23:54:54,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694233 2024-11-22T23:54:54,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694233 2024-11-22T23:54:54,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694233) roll requested 2024-11-22T23:54:54,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694242 2024-11-22T23:54:54,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694242 2024-11-22T23:54:54,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694242) roll requested 2024-11-22T23:54:54,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694251 2024-11-22T23:54:54,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694251 2024-11-22T23:54:54,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694251) roll requested 2024-11-22T23:54:54,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694260 2024-11-22T23:54:54,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694260 2024-11-22T23:54:54,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694260) roll requested 2024-11-22T23:54:54,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694268 2024-11-22T23:54:54,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694268 2024-11-22T23:54:54,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694268) roll requested 2024-11-22T23:54:54,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694278 2024-11-22T23:54:54,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694278 2024-11-22T23:54:54,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694278) roll requested 2024-11-22T23:54:54,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694289 2024-11-22T23:54:54,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694289 2024-11-22T23:54:54,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694289) roll requested 2024-11-22T23:54:54,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694298 2024-11-22T23:54:54,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694298 2024-11-22T23:54:54,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694298) roll requested 2024-11-22T23:54:54,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694306 2024-11-22T23:54:54,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694306 2024-11-22T23:54:54,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694306) roll requested 2024-11-22T23:54:54,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694323 2024-11-22T23:54:54,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694323 2024-11-22T23:54:54,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694323) roll requested 2024-11-22T23:54:54,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694334 2024-11-22T23:54:54,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694334 2024-11-22T23:54:54,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694334) roll requested 2024-11-22T23:54:54,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694342 2024-11-22T23:54:54,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694342 2024-11-22T23:54:54,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694342) roll requested 2024-11-22T23:54:54,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694350 2024-11-22T23:54:54,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694350 2024-11-22T23:54:54,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694350) roll requested 2024-11-22T23:54:54,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694359 2024-11-22T23:54:54,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694359 2024-11-22T23:54:54,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694359) roll requested 2024-11-22T23:54:54,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694367 2024-11-22T23:54:54,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694367 2024-11-22T23:54:54,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694367) roll requested 2024-11-22T23:54:54,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694375 2024-11-22T23:54:54,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694375 2024-11-22T23:54:54,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694375) roll requested 2024-11-22T23:54:54,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694383 2024-11-22T23:54:54,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694383 2024-11-22T23:54:54,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694383) roll requested 2024-11-22T23:54:54,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694391 2024-11-22T23:54:54,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694391 2024-11-22T23:54:54,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694391) roll requested 2024-11-22T23:54:54,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694402 2024-11-22T23:54:54,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694402 2024-11-22T23:54:54,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694402) roll requested 2024-11-22T23:54:54,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694410 2024-11-22T23:54:54,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694410 2024-11-22T23:54:54,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694410) roll requested 2024-11-22T23:54:54,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694419 2024-11-22T23:54:54,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694419 2024-11-22T23:54:54,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694419) roll requested 2024-11-22T23:54:54,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694427 2024-11-22T23:54:54,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694427 2024-11-22T23:54:54,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694427) roll requested 2024-11-22T23:54:54,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694438 2024-11-22T23:54:54,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694438 2024-11-22T23:54:54,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694438) roll requested 2024-11-22T23:54:54,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694446 2024-11-22T23:54:54,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694446 2024-11-22T23:54:54,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694446) roll requested 2024-11-22T23:54:54,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694454 2024-11-22T23:54:54,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694454 2024-11-22T23:54:54,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694454) roll requested 2024-11-22T23:54:54,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694463 2024-11-22T23:54:54,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694463 2024-11-22T23:54:54,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694463) roll requested 2024-11-22T23:54:54,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694472 2024-11-22T23:54:54,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694472 2024-11-22T23:54:54,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694472) roll requested 2024-11-22T23:54:54,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694480 2024-11-22T23:54:54,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694480 2024-11-22T23:54:54,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694480) roll requested 2024-11-22T23:54:54,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694487 2024-11-22T23:54:54,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694487 2024-11-22T23:54:54,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694487) roll requested 2024-11-22T23:54:54,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694495 2024-11-22T23:54:54,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694495 2024-11-22T23:54:54,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694495) roll requested 2024-11-22T23:54:54,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694503 2024-11-22T23:54:54,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694503 2024-11-22T23:54:54,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694503) roll requested 2024-11-22T23:54:54,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694511 2024-11-22T23:54:54,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694511 2024-11-22T23:54:54,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694511) roll requested 2024-11-22T23:54:54,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694519 2024-11-22T23:54:54,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694519 2024-11-22T23:54:54,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694519) roll requested 2024-11-22T23:54:54,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694528 2024-11-22T23:54:54,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694528 2024-11-22T23:54:54,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694528) roll requested 2024-11-22T23:54:54,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694537 2024-11-22T23:54:54,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694537 2024-11-22T23:54:54,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694537) roll requested 2024-11-22T23:54:54,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694545 2024-11-22T23:54:54,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694545 2024-11-22T23:54:54,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694545) roll requested 2024-11-22T23:54:54,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694553 2024-11-22T23:54:54,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694553 2024-11-22T23:54:54,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694553) roll requested 2024-11-22T23:54:54,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694561 2024-11-22T23:54:54,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694561 2024-11-22T23:54:54,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694561) roll requested 2024-11-22T23:54:54,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694570 2024-11-22T23:54:54,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694570 2024-11-22T23:54:54,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694570) roll requested 2024-11-22T23:54:54,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694578 2024-11-22T23:54:54,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694578 2024-11-22T23:54:54,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694578) roll requested 2024-11-22T23:54:54,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694586 2024-11-22T23:54:54,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694586 2024-11-22T23:54:54,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694586) roll requested 2024-11-22T23:54:54,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694594 2024-11-22T23:54:54,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694594 2024-11-22T23:54:54,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694594) roll requested 2024-11-22T23:54:54,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694602 2024-11-22T23:54:54,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694602 2024-11-22T23:54:54,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694602) roll requested 2024-11-22T23:54:54,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694610 2024-11-22T23:54:54,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694610 2024-11-22T23:54:54,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694610) roll requested 2024-11-22T23:54:54,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694618 2024-11-22T23:54:54,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694618 2024-11-22T23:54:54,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694618) roll requested 2024-11-22T23:54:54,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694631 2024-11-22T23:54:54,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694631 2024-11-22T23:54:54,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694631) roll requested 2024-11-22T23:54:54,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694639 2024-11-22T23:54:54,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694639 2024-11-22T23:54:54,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694639) roll requested 2024-11-22T23:54:54,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694648 2024-11-22T23:54:54,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694648 2024-11-22T23:54:54,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694648) roll requested 2024-11-22T23:54:54,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694664 2024-11-22T23:54:54,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694664 2024-11-22T23:54:54,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694664) roll requested 2024-11-22T23:54:54,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694673 2024-11-22T23:54:54,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694673 2024-11-22T23:54:54,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694673) roll requested 2024-11-22T23:54:54,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694682 2024-11-22T23:54:54,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694682 2024-11-22T23:54:54,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694682) roll requested 2024-11-22T23:54:54,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694690 2024-11-22T23:54:54,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694690 2024-11-22T23:54:54,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694690) roll requested 2024-11-22T23:54:54,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694698 2024-11-22T23:54:54,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694698 2024-11-22T23:54:54,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694698) roll requested 2024-11-22T23:54:54,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694706 2024-11-22T23:54:54,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694706 2024-11-22T23:54:54,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694706) roll requested 2024-11-22T23:54:54,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694713 2024-11-22T23:54:54,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694713 2024-11-22T23:54:54,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694713) roll requested 2024-11-22T23:54:54,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694721 2024-11-22T23:54:54,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694721 2024-11-22T23:54:54,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694721) roll requested 2024-11-22T23:54:54,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694729 2024-11-22T23:54:54,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694729 2024-11-22T23:54:54,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694729) roll requested 2024-11-22T23:54:54,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694738 2024-11-22T23:54:54,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694738 2024-11-22T23:54:54,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694738) roll requested 2024-11-22T23:54:54,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694746 2024-11-22T23:54:54,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694746 2024-11-22T23:54:54,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694746) roll requested 2024-11-22T23:54:54,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694754 2024-11-22T23:54:54,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694754 2024-11-22T23:54:54,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694754) roll requested 2024-11-22T23:54:54,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694766 2024-11-22T23:54:54,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694766 2024-11-22T23:54:54,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694766) roll requested 2024-11-22T23:54:54,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694775 2024-11-22T23:54:54,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694775 2024-11-22T23:54:54,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694775) roll requested 2024-11-22T23:54:54,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694783 2024-11-22T23:54:54,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694783 2024-11-22T23:54:54,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694783) roll requested 2024-11-22T23:54:54,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694791 2024-11-22T23:54:54,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694791 2024-11-22T23:54:54,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694791) roll requested 2024-11-22T23:54:54,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694799 2024-11-22T23:54:54,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694799 2024-11-22T23:54:54,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694799) roll requested 2024-11-22T23:54:54,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694808 2024-11-22T23:54:54,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694808 2024-11-22T23:54:54,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694808) roll requested 2024-11-22T23:54:54,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694816 2024-11-22T23:54:54,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694816 2024-11-22T23:54:54,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694816) roll requested 2024-11-22T23:54:54,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694824 2024-11-22T23:54:54,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694824 2024-11-22T23:54:54,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694824) roll requested 2024-11-22T23:54:54,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694833 2024-11-22T23:54:54,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694833 2024-11-22T23:54:54,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694833) roll requested 2024-11-22T23:54:54,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694842 2024-11-22T23:54:54,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694842 2024-11-22T23:54:54,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694842) roll requested 2024-11-22T23:54:54,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694850 2024-11-22T23:54:54,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694850 2024-11-22T23:54:54,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694850) roll requested 2024-11-22T23:54:54,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694858 2024-11-22T23:54:54,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694858 2024-11-22T23:54:54,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694858) roll requested 2024-11-22T23:54:54,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694873 2024-11-22T23:54:54,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694873 2024-11-22T23:54:54,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694873) roll requested 2024-11-22T23:54:54,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694882 2024-11-22T23:54:54,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694882 2024-11-22T23:54:54,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694882) roll requested 2024-11-22T23:54:54,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694889 2024-11-22T23:54:54,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694889 2024-11-22T23:54:54,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694889) roll requested 2024-11-22T23:54:54,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694898 2024-11-22T23:54:54,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694898 2024-11-22T23:54:54,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694898) roll requested 2024-11-22T23:54:54,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694905 2024-11-22T23:54:54,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694905 2024-11-22T23:54:54,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694905) roll requested 2024-11-22T23:54:54,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694918 2024-11-22T23:54:54,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694918 2024-11-22T23:54:54,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694918) roll requested 2024-11-22T23:54:54,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694926 2024-11-22T23:54:54,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694926 2024-11-22T23:54:54,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694926) roll requested 2024-11-22T23:54:54,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694934 2024-11-22T23:54:54,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694934 2024-11-22T23:54:54,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694934) roll requested 2024-11-22T23:54:54,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694942 2024-11-22T23:54:54,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694942 2024-11-22T23:54:54,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694942) roll requested 2024-11-22T23:54:54,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694950 2024-11-22T23:54:54,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694950 2024-11-22T23:54:54,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694950) roll requested 2024-11-22T23:54:54,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694959 2024-11-22T23:54:54,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694959 2024-11-22T23:54:54,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694959) roll requested 2024-11-22T23:54:54,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694970 2024-11-22T23:54:54,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:54,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694970 2024-11-22T23:54:54,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:54,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:54,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694970) roll requested 2024-11-22T23:54:54,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319694991 2024-11-22T23:54:55,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694991 2024-11-22T23:54:55,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319694991) roll requested 2024-11-22T23:54:55,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695003 2024-11-22T23:54:55,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319694991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695003 2024-11-22T23:54:55,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695003) roll requested 2024-11-22T23:54:55,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695013 2024-11-22T23:54:55,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695013 2024-11-22T23:54:55,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695013) roll requested 2024-11-22T23:54:55,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695021 2024-11-22T23:54:55,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695021 2024-11-22T23:54:55,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695021) roll requested 2024-11-22T23:54:55,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695030 2024-11-22T23:54:55,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695030 2024-11-22T23:54:55,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695030) roll requested 2024-11-22T23:54:55,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695041 2024-11-22T23:54:55,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695041 2024-11-22T23:54:55,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695041) roll requested 2024-11-22T23:54:55,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695049 2024-11-22T23:54:55,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695049 2024-11-22T23:54:55,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695049) roll requested 2024-11-22T23:54:55,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695056 2024-11-22T23:54:55,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695056 2024-11-22T23:54:55,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695056) roll requested 2024-11-22T23:54:55,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695074 2024-11-22T23:54:55,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695074 2024-11-22T23:54:55,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695074) roll requested 2024-11-22T23:54:55,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695089 2024-11-22T23:54:55,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695089 2024-11-22T23:54:55,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695089) roll requested 2024-11-22T23:54:55,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695101 2024-11-22T23:54:55,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695101 2024-11-22T23:54:55,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695101) roll requested 2024-11-22T23:54:55,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695109 2024-11-22T23:54:55,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695109 2024-11-22T23:54:55,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695109) roll requested 2024-11-22T23:54:55,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695118 2024-11-22T23:54:55,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695118 2024-11-22T23:54:55,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695118) roll requested 2024-11-22T23:54:55,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695128 2024-11-22T23:54:55,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695128 2024-11-22T23:54:55,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695128) roll requested 2024-11-22T23:54:55,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695135 2024-11-22T23:54:55,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695135 2024-11-22T23:54:55,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695135) roll requested 2024-11-22T23:54:55,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695143 2024-11-22T23:54:55,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695143 2024-11-22T23:54:55,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695143) roll requested 2024-11-22T23:54:55,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695152 2024-11-22T23:54:55,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695152 2024-11-22T23:54:55,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695152) roll requested 2024-11-22T23:54:55,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695160 2024-11-22T23:54:55,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695160 2024-11-22T23:54:55,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695160) roll requested 2024-11-22T23:54:55,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695170 2024-11-22T23:54:55,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695170 2024-11-22T23:54:55,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695170) roll requested 2024-11-22T23:54:55,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695203 2024-11-22T23:54:55,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695203 2024-11-22T23:54:55,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695203) roll requested 2024-11-22T23:54:55,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695219 2024-11-22T23:54:55,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695219 2024-11-22T23:54:55,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695219) roll requested 2024-11-22T23:54:55,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695229 2024-11-22T23:54:55,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695229 2024-11-22T23:54:55,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695229) roll requested 2024-11-22T23:54:55,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695238 2024-11-22T23:54:55,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695238 2024-11-22T23:54:55,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695238) roll requested 2024-11-22T23:54:55,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695246 2024-11-22T23:54:55,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695246 2024-11-22T23:54:55,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695246) roll requested 2024-11-22T23:54:55,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695254 2024-11-22T23:54:55,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695254 2024-11-22T23:54:55,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695254) roll requested 2024-11-22T23:54:55,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695262 2024-11-22T23:54:55,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695262 2024-11-22T23:54:55,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695262) roll requested 2024-11-22T23:54:55,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695270 2024-11-22T23:54:55,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695270 2024-11-22T23:54:55,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695270) roll requested 2024-11-22T23:54:55,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695278 2024-11-22T23:54:55,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695278 2024-11-22T23:54:55,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695278) roll requested 2024-11-22T23:54:55,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695286 2024-11-22T23:54:55,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695286 2024-11-22T23:54:55,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695286) roll requested 2024-11-22T23:54:55,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695294 2024-11-22T23:54:55,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695294 2024-11-22T23:54:55,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695294) roll requested 2024-11-22T23:54:55,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695301 2024-11-22T23:54:55,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695301 2024-11-22T23:54:55,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695301) roll requested 2024-11-22T23:54:55,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695322 2024-11-22T23:54:55,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695322 2024-11-22T23:54:55,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695322) roll requested 2024-11-22T23:54:55,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695333 2024-11-22T23:54:55,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695333 2024-11-22T23:54:55,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695333) roll requested 2024-11-22T23:54:55,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695341 2024-11-22T23:54:55,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695341 2024-11-22T23:54:55,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695341) roll requested 2024-11-22T23:54:55,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695354 2024-11-22T23:54:55,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695354 2024-11-22T23:54:55,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695354) roll requested 2024-11-22T23:54:55,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695363 2024-11-22T23:54:55,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695363 2024-11-22T23:54:55,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695363) roll requested 2024-11-22T23:54:55,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695374 2024-11-22T23:54:55,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695374 2024-11-22T23:54:55,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695374) roll requested 2024-11-22T23:54:55,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695382 2024-11-22T23:54:55,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695382 2024-11-22T23:54:55,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695382) roll requested 2024-11-22T23:54:55,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695390 2024-11-22T23:54:55,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695390 2024-11-22T23:54:55,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695390) roll requested 2024-11-22T23:54:55,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695398 2024-11-22T23:54:55,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695398 2024-11-22T23:54:55,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695398) roll requested 2024-11-22T23:54:55,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695407 2024-11-22T23:54:55,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695407 2024-11-22T23:54:55,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695407) roll requested 2024-11-22T23:54:55,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695417 2024-11-22T23:54:55,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695417 2024-11-22T23:54:55,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695417) roll requested 2024-11-22T23:54:55,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695426 2024-11-22T23:54:55,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695426 2024-11-22T23:54:55,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695426) roll requested 2024-11-22T23:54:55,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695434 2024-11-22T23:54:55,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695434 2024-11-22T23:54:55,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695434) roll requested 2024-11-22T23:54:55,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695441 2024-11-22T23:54:55,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695441 2024-11-22T23:54:55,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695441) roll requested 2024-11-22T23:54:55,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695449 2024-11-22T23:54:55,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695449 2024-11-22T23:54:55,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695449) roll requested 2024-11-22T23:54:55,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695462 2024-11-22T23:54:55,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695462 2024-11-22T23:54:55,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695462) roll requested 2024-11-22T23:54:55,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695472 2024-11-22T23:54:55,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695472 2024-11-22T23:54:55,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695472) roll requested 2024-11-22T23:54:55,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695480 2024-11-22T23:54:55,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695480 2024-11-22T23:54:55,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695480) roll requested 2024-11-22T23:54:55,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695491 2024-11-22T23:54:55,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695491 2024-11-22T23:54:55,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695491) roll requested 2024-11-22T23:54:55,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695505 2024-11-22T23:54:55,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695505 2024-11-22T23:54:55,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695505) roll requested 2024-11-22T23:54:55,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695515 2024-11-22T23:54:55,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695515 2024-11-22T23:54:55,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695515) roll requested 2024-11-22T23:54:55,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695534 2024-11-22T23:54:55,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695534 2024-11-22T23:54:55,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695534) roll requested 2024-11-22T23:54:55,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695544 2024-11-22T23:54:55,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695544 2024-11-22T23:54:55,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695544) roll requested 2024-11-22T23:54:55,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695552 2024-11-22T23:54:55,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695552 2024-11-22T23:54:55,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695552) roll requested 2024-11-22T23:54:55,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695560 2024-11-22T23:54:55,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695560 2024-11-22T23:54:55,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695560) roll requested 2024-11-22T23:54:55,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695569 2024-11-22T23:54:55,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695569 2024-11-22T23:54:55,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695569) roll requested 2024-11-22T23:54:55,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695577 2024-11-22T23:54:55,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695577 2024-11-22T23:54:55,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695577) roll requested 2024-11-22T23:54:55,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695584 2024-11-22T23:54:55,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695584 2024-11-22T23:54:55,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695584) roll requested 2024-11-22T23:54:55,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695593 2024-11-22T23:54:55,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695593 2024-11-22T23:54:55,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695593) roll requested 2024-11-22T23:54:55,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695600 2024-11-22T23:54:55,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695600 2024-11-22T23:54:55,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695600) roll requested 2024-11-22T23:54:55,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695609 2024-11-22T23:54:55,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695609 2024-11-22T23:54:55,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695609) roll requested 2024-11-22T23:54:55,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695617 2024-11-22T23:54:55,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695617 2024-11-22T23:54:55,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695617) roll requested 2024-11-22T23:54:55,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695626 2024-11-22T23:54:55,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695626 2024-11-22T23:54:55,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695626) roll requested 2024-11-22T23:54:55,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695635 2024-11-22T23:54:55,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695635 2024-11-22T23:54:55,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695635) roll requested 2024-11-22T23:54:55,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695654 2024-11-22T23:54:55,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695654 2024-11-22T23:54:55,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695654) roll requested 2024-11-22T23:54:55,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695665 2024-11-22T23:54:55,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695665 2024-11-22T23:54:55,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695665) roll requested 2024-11-22T23:54:55,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695680 2024-11-22T23:54:55,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695680 2024-11-22T23:54:55,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695680) roll requested 2024-11-22T23:54:55,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695689 2024-11-22T23:54:55,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695689 2024-11-22T23:54:55,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695689) roll requested 2024-11-22T23:54:55,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695702 2024-11-22T23:54:55,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695702 2024-11-22T23:54:55,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695702) roll requested 2024-11-22T23:54:55,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695711 2024-11-22T23:54:55,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695711 2024-11-22T23:54:55,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695711) roll requested 2024-11-22T23:54:55,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695719 2024-11-22T23:54:55,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695719 2024-11-22T23:54:55,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695719) roll requested 2024-11-22T23:54:55,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695733 2024-11-22T23:54:55,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695733 2024-11-22T23:54:55,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695733) roll requested 2024-11-22T23:54:55,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695742 2024-11-22T23:54:55,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695742 2024-11-22T23:54:55,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695742) roll requested 2024-11-22T23:54:55,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695751 2024-11-22T23:54:55,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695751 2024-11-22T23:54:55,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695751) roll requested 2024-11-22T23:54:55,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695760 2024-11-22T23:54:55,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695760 2024-11-22T23:54:55,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695760) roll requested 2024-11-22T23:54:55,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695769 2024-11-22T23:54:55,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695769 2024-11-22T23:54:55,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695769) roll requested 2024-11-22T23:54:55,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695781 2024-11-22T23:54:55,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695781 2024-11-22T23:54:55,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695781) roll requested 2024-11-22T23:54:55,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695790 2024-11-22T23:54:55,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695790 2024-11-22T23:54:55,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695790) roll requested 2024-11-22T23:54:55,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695805 2024-11-22T23:54:55,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695805 2024-11-22T23:54:55,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695805) roll requested 2024-11-22T23:54:55,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695813 2024-11-22T23:54:55,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695813 2024-11-22T23:54:55,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695813) roll requested 2024-11-22T23:54:55,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695822 2024-11-22T23:54:55,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695822 2024-11-22T23:54:55,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695822) roll requested 2024-11-22T23:54:55,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695830 2024-11-22T23:54:55,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695830 2024-11-22T23:54:55,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695830) roll requested 2024-11-22T23:54:55,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695838 2024-11-22T23:54:55,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695838 2024-11-22T23:54:55,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695838) roll requested 2024-11-22T23:54:55,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695849 2024-11-22T23:54:55,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695849 2024-11-22T23:54:55,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695849) roll requested 2024-11-22T23:54:55,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695857 2024-11-22T23:54:55,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695857 2024-11-22T23:54:55,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695857) roll requested 2024-11-22T23:54:55,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695865 2024-11-22T23:54:55,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695865 2024-11-22T23:54:55,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695865) roll requested 2024-11-22T23:54:55,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695874 2024-11-22T23:54:55,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695874 2024-11-22T23:54:55,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695874) roll requested 2024-11-22T23:54:55,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695882 2024-11-22T23:54:55,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695882 2024-11-22T23:54:55,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695882) roll requested 2024-11-22T23:54:55,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695893 2024-11-22T23:54:55,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695893 2024-11-22T23:54:55,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695893) roll requested 2024-11-22T23:54:55,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695901 2024-11-22T23:54:55,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695901 2024-11-22T23:54:55,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695901) roll requested 2024-11-22T23:54:55,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695908 2024-11-22T23:54:55,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695908 2024-11-22T23:54:55,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695908) roll requested 2024-11-22T23:54:55,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695916 2024-11-22T23:54:55,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695916 2024-11-22T23:54:55,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695916) roll requested 2024-11-22T23:54:55,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695924 2024-11-22T23:54:55,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695924 2024-11-22T23:54:55,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695924) roll requested 2024-11-22T23:54:55,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695932 2024-11-22T23:54:55,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695932 2024-11-22T23:54:55,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695932) roll requested 2024-11-22T23:54:55,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695940 2024-11-22T23:54:55,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695940 2024-11-22T23:54:55,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695940) roll requested 2024-11-22T23:54:55,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695948 2024-11-22T23:54:55,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695948 2024-11-22T23:54:55,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695948) roll requested 2024-11-22T23:54:55,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695956 2024-11-22T23:54:55,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695956 2024-11-22T23:54:55,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695956) roll requested 2024-11-22T23:54:55,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695964 2024-11-22T23:54:55,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695964 2024-11-22T23:54:55,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695964) roll requested 2024-11-22T23:54:55,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695971 2024-11-22T23:54:55,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695971 2024-11-22T23:54:55,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695971) roll requested 2024-11-22T23:54:55,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695980 2024-11-22T23:54:55,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:55,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695980 2024-11-22T23:54:55,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:55,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:55,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695980) roll requested 2024-11-22T23:54:55,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319695988 2024-11-22T23:54:56,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695988 2024-11-22T23:54:56,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319695988) roll requested 2024-11-22T23:54:56,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696003 2024-11-22T23:54:56,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319695988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696003 2024-11-22T23:54:56,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696003) roll requested 2024-11-22T23:54:56,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696011 2024-11-22T23:54:56,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696011 2024-11-22T23:54:56,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696011) roll requested 2024-11-22T23:54:56,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696019 2024-11-22T23:54:56,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696019 2024-11-22T23:54:56,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696019) roll requested 2024-11-22T23:54:56,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696032 2024-11-22T23:54:56,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696032 2024-11-22T23:54:56,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696032) roll requested 2024-11-22T23:54:56,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696045 2024-11-22T23:54:56,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696045 2024-11-22T23:54:56,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696045) roll requested 2024-11-22T23:54:56,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696057 2024-11-22T23:54:56,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696057 2024-11-22T23:54:56,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696057) roll requested 2024-11-22T23:54:56,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696081 2024-11-22T23:54:56,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696081 2024-11-22T23:54:56,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696081) roll requested 2024-11-22T23:54:56,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696091 2024-11-22T23:54:56,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696091 2024-11-22T23:54:56,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696091) roll requested 2024-11-22T23:54:56,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696101 2024-11-22T23:54:56,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696101 2024-11-22T23:54:56,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696101) roll requested 2024-11-22T23:54:56,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696111 2024-11-22T23:54:56,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696111 2024-11-22T23:54:56,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696111) roll requested 2024-11-22T23:54:56,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696120 2024-11-22T23:54:56,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696120 2024-11-22T23:54:56,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696120) roll requested 2024-11-22T23:54:56,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696151 2024-11-22T23:54:56,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696151 2024-11-22T23:54:56,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696151) roll requested 2024-11-22T23:54:56,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696160 2024-11-22T23:54:56,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696160 2024-11-22T23:54:56,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696160) roll requested 2024-11-22T23:54:56,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696167 2024-11-22T23:54:56,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696167 2024-11-22T23:54:56,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696167) roll requested 2024-11-22T23:54:56,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696175 2024-11-22T23:54:56,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696175 2024-11-22T23:54:56,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696175) roll requested 2024-11-22T23:54:56,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696184 2024-11-22T23:54:56,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696184 2024-11-22T23:54:56,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696184) roll requested 2024-11-22T23:54:56,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696191 2024-11-22T23:54:56,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696191 2024-11-22T23:54:56,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696191) roll requested 2024-11-22T23:54:56,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696199 2024-11-22T23:54:56,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696199 2024-11-22T23:54:56,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696199) roll requested 2024-11-22T23:54:56,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696216 2024-11-22T23:54:56,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696216 2024-11-22T23:54:56,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696216) roll requested 2024-11-22T23:54:56,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696224 2024-11-22T23:54:56,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696224 2024-11-22T23:54:56,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696224) roll requested 2024-11-22T23:54:56,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696232 2024-11-22T23:54:56,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696232 2024-11-22T23:54:56,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696232) roll requested 2024-11-22T23:54:56,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696242 2024-11-22T23:54:56,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696242 2024-11-22T23:54:56,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696242) roll requested 2024-11-22T23:54:56,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696249 2024-11-22T23:54:56,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696249 2024-11-22T23:54:56,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696249) roll requested 2024-11-22T23:54:56,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696257 2024-11-22T23:54:56,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696257 2024-11-22T23:54:56,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696257) roll requested 2024-11-22T23:54:56,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696264 2024-11-22T23:54:56,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696264 2024-11-22T23:54:56,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696264) roll requested 2024-11-22T23:54:56,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696271 2024-11-22T23:54:56,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696271 2024-11-22T23:54:56,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696271) roll requested 2024-11-22T23:54:56,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696278 2024-11-22T23:54:56,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696278 2024-11-22T23:54:56,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696278) roll requested 2024-11-22T23:54:56,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696286 2024-11-22T23:54:56,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696286 2024-11-22T23:54:56,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696286) roll requested 2024-11-22T23:54:56,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696293 2024-11-22T23:54:56,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696293 2024-11-22T23:54:56,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696293) roll requested 2024-11-22T23:54:56,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696301 2024-11-22T23:54:56,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696301 2024-11-22T23:54:56,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696301) roll requested 2024-11-22T23:54:56,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696308 2024-11-22T23:54:56,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696308 2024-11-22T23:54:56,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696308) roll requested 2024-11-22T23:54:56,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696315 2024-11-22T23:54:56,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696315 2024-11-22T23:54:56,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696315) roll requested 2024-11-22T23:54:56,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696323 2024-11-22T23:54:56,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696323 2024-11-22T23:54:56,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696323) roll requested 2024-11-22T23:54:56,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696332 2024-11-22T23:54:56,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696332 2024-11-22T23:54:56,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696332) roll requested 2024-11-22T23:54:56,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696339 2024-11-22T23:54:56,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696339 2024-11-22T23:54:56,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696339) roll requested 2024-11-22T23:54:56,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696347 2024-11-22T23:54:56,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696347 2024-11-22T23:54:56,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696347) roll requested 2024-11-22T23:54:56,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696354 2024-11-22T23:54:56,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696354 2024-11-22T23:54:56,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696354) roll requested 2024-11-22T23:54:56,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696362 2024-11-22T23:54:56,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696362 2024-11-22T23:54:56,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696362) roll requested 2024-11-22T23:54:56,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696369 2024-11-22T23:54:56,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696369 2024-11-22T23:54:56,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696369) roll requested 2024-11-22T23:54:56,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696377 2024-11-22T23:54:56,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696377 2024-11-22T23:54:56,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696377) roll requested 2024-11-22T23:54:56,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696385 2024-11-22T23:54:56,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696385 2024-11-22T23:54:56,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696385) roll requested 2024-11-22T23:54:56,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696401 2024-11-22T23:54:56,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696401 2024-11-22T23:54:56,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696401) roll requested 2024-11-22T23:54:56,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696413 2024-11-22T23:54:56,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696413 2024-11-22T23:54:56,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696413) roll requested 2024-11-22T23:54:56,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696428 2024-11-22T23:54:56,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696428 2024-11-22T23:54:56,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696428) roll requested 2024-11-22T23:54:56,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696437 2024-11-22T23:54:56,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696437 2024-11-22T23:54:56,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696437) roll requested 2024-11-22T23:54:56,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696445 2024-11-22T23:54:56,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696445 2024-11-22T23:54:56,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696445) roll requested 2024-11-22T23:54:56,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696456 2024-11-22T23:54:56,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696456 2024-11-22T23:54:56,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696456) roll requested 2024-11-22T23:54:56,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696463 2024-11-22T23:54:56,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696463 2024-11-22T23:54:56,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696463) roll requested 2024-11-22T23:54:56,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696471 2024-11-22T23:54:56,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696471 2024-11-22T23:54:56,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696471) roll requested 2024-11-22T23:54:56,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696479 2024-11-22T23:54:56,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696479 2024-11-22T23:54:56,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696479) roll requested 2024-11-22T23:54:56,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696487 2024-11-22T23:54:56,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696487 2024-11-22T23:54:56,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696487) roll requested 2024-11-22T23:54:56,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696494 2024-11-22T23:54:56,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696494 2024-11-22T23:54:56,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696494) roll requested 2024-11-22T23:54:56,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696506 2024-11-22T23:54:56,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696506 2024-11-22T23:54:56,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696506) roll requested 2024-11-22T23:54:56,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696514 2024-11-22T23:54:56,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696514 2024-11-22T23:54:56,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696514) roll requested 2024-11-22T23:54:56,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696522 2024-11-22T23:54:56,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696522 2024-11-22T23:54:56,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696522) roll requested 2024-11-22T23:54:56,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696529 2024-11-22T23:54:56,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696529 2024-11-22T23:54:56,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696529) roll requested 2024-11-22T23:54:56,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696536 2024-11-22T23:54:56,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696536 2024-11-22T23:54:56,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696536) roll requested 2024-11-22T23:54:56,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696547 2024-11-22T23:54:56,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696547 2024-11-22T23:54:56,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696547) roll requested 2024-11-22T23:54:56,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696565 2024-11-22T23:54:56,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696565 2024-11-22T23:54:56,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696565) roll requested 2024-11-22T23:54:56,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696573 2024-11-22T23:54:56,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696573 2024-11-22T23:54:56,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696573) roll requested 2024-11-22T23:54:56,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696581 2024-11-22T23:54:56,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696581 2024-11-22T23:54:56,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696581) roll requested 2024-11-22T23:54:56,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696609 2024-11-22T23:54:56,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696609 2024-11-22T23:54:56,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696609) roll requested 2024-11-22T23:54:56,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696618 2024-11-22T23:54:56,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696618 2024-11-22T23:54:56,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696618) roll requested 2024-11-22T23:54:56,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696629 2024-11-22T23:54:56,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696629 2024-11-22T23:54:56,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696629) roll requested 2024-11-22T23:54:56,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696641 2024-11-22T23:54:56,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696641 2024-11-22T23:54:56,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696641) roll requested 2024-11-22T23:54:56,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696652 2024-11-22T23:54:56,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696652 2024-11-22T23:54:56,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696652) roll requested 2024-11-22T23:54:56,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696661 2024-11-22T23:54:56,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696661 2024-11-22T23:54:56,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696661) roll requested 2024-11-22T23:54:56,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696677 2024-11-22T23:54:56,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696677 2024-11-22T23:54:56,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696677) roll requested 2024-11-22T23:54:56,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696687 2024-11-22T23:54:56,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696687 2024-11-22T23:54:56,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696687) roll requested 2024-11-22T23:54:56,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696704 2024-11-22T23:54:56,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696704 2024-11-22T23:54:56,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696704) roll requested 2024-11-22T23:54:56,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696711 2024-11-22T23:54:56,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696711 2024-11-22T23:54:56,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696711) roll requested 2024-11-22T23:54:56,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696719 2024-11-22T23:54:56,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696719 2024-11-22T23:54:56,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696719) roll requested 2024-11-22T23:54:56,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696727 2024-11-22T23:54:56,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696727 2024-11-22T23:54:56,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696727) roll requested 2024-11-22T23:54:56,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696735 2024-11-22T23:54:56,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696735 2024-11-22T23:54:56,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696735) roll requested 2024-11-22T23:54:56,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696745 2024-11-22T23:54:56,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696745 2024-11-22T23:54:56,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696745) roll requested 2024-11-22T23:54:56,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696759 2024-11-22T23:54:56,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696759 2024-11-22T23:54:56,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696759) roll requested 2024-11-22T23:54:56,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696767 2024-11-22T23:54:56,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696767 2024-11-22T23:54:56,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696767) roll requested 2024-11-22T23:54:56,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696774 2024-11-22T23:54:56,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696774 2024-11-22T23:54:56,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696774) roll requested 2024-11-22T23:54:56,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696782 2024-11-22T23:54:56,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696782 2024-11-22T23:54:56,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696782) roll requested 2024-11-22T23:54:56,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696790 2024-11-22T23:54:56,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696790 2024-11-22T23:54:56,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696790) roll requested 2024-11-22T23:54:56,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696799 2024-11-22T23:54:56,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696799 2024-11-22T23:54:56,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696799) roll requested 2024-11-22T23:54:56,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696806 2024-11-22T23:54:56,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696806 2024-11-22T23:54:56,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696806) roll requested 2024-11-22T23:54:56,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696813 2024-11-22T23:54:56,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696813 2024-11-22T23:54:56,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696813) roll requested 2024-11-22T23:54:56,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696822 2024-11-22T23:54:56,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696822 2024-11-22T23:54:56,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696822) roll requested 2024-11-22T23:54:56,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696829 2024-11-22T23:54:56,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696829 2024-11-22T23:54:56,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696829) roll requested 2024-11-22T23:54:56,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696837 2024-11-22T23:54:56,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696837 2024-11-22T23:54:56,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696837) roll requested 2024-11-22T23:54:56,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696846 2024-11-22T23:54:56,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696846 2024-11-22T23:54:56,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696846) roll requested 2024-11-22T23:54:56,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696854 2024-11-22T23:54:56,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696854 2024-11-22T23:54:56,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696854) roll requested 2024-11-22T23:54:56,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696862 2024-11-22T23:54:56,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696862 2024-11-22T23:54:56,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696862) roll requested 2024-11-22T23:54:56,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696870 2024-11-22T23:54:56,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696870 2024-11-22T23:54:56,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696870) roll requested 2024-11-22T23:54:56,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696878 2024-11-22T23:54:56,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696878 2024-11-22T23:54:56,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696878) roll requested 2024-11-22T23:54:56,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696885 2024-11-22T23:54:56,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696885 2024-11-22T23:54:56,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696885) roll requested 2024-11-22T23:54:56,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696892 2024-11-22T23:54:56,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696892 2024-11-22T23:54:56,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696892) roll requested 2024-11-22T23:54:56,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696901 2024-11-22T23:54:56,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696901 2024-11-22T23:54:56,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696901) roll requested 2024-11-22T23:54:56,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696909 2024-11-22T23:54:56,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696909 2024-11-22T23:54:56,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696909) roll requested 2024-11-22T23:54:56,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696916 2024-11-22T23:54:56,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696916 2024-11-22T23:54:56,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696916) roll requested 2024-11-22T23:54:56,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696923 2024-11-22T23:54:56,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696923 2024-11-22T23:54:56,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696923) roll requested 2024-11-22T23:54:56,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696930 2024-11-22T23:54:56,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696930 2024-11-22T23:54:56,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696930) roll requested 2024-11-22T23:54:56,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696937 2024-11-22T23:54:56,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696937 2024-11-22T23:54:56,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696937) roll requested 2024-11-22T23:54:56,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696944 2024-11-22T23:54:56,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696944 2024-11-22T23:54:56,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696944) roll requested 2024-11-22T23:54:56,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696952 2024-11-22T23:54:56,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696952 2024-11-22T23:54:56,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696952) roll requested 2024-11-22T23:54:56,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696960 2024-11-22T23:54:56,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696960 2024-11-22T23:54:56,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696960) roll requested 2024-11-22T23:54:56,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696972 2024-11-22T23:54:56,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696972 2024-11-22T23:54:56,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696972) roll requested 2024-11-22T23:54:56,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696980 2024-11-22T23:54:56,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:56,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696980 2024-11-22T23:54:56,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:56,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:56,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696980) roll requested 2024-11-22T23:54:56,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319696993 2024-11-22T23:54:57,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696993 2024-11-22T23:54:57,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319696993) roll requested 2024-11-22T23:54:57,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697001 2024-11-22T23:54:57,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319696993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697001 2024-11-22T23:54:57,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697001) roll requested 2024-11-22T23:54:57,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697009 2024-11-22T23:54:57,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697009 2024-11-22T23:54:57,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697009) roll requested 2024-11-22T23:54:57,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697019 2024-11-22T23:54:57,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697019 2024-11-22T23:54:57,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697019) roll requested 2024-11-22T23:54:57,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697026 2024-11-22T23:54:57,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697026 2024-11-22T23:54:57,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697026) roll requested 2024-11-22T23:54:57,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697034 2024-11-22T23:54:57,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697034 2024-11-22T23:54:57,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697034) roll requested 2024-11-22T23:54:57,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697042 2024-11-22T23:54:57,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697042 2024-11-22T23:54:57,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697042) roll requested 2024-11-22T23:54:57,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697053 2024-11-22T23:54:57,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697053 2024-11-22T23:54:57,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697053) roll requested 2024-11-22T23:54:57,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697063 2024-11-22T23:54:57,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697063 2024-11-22T23:54:57,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697063) roll requested 2024-11-22T23:54:57,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697071 2024-11-22T23:54:57,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697071 2024-11-22T23:54:57,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697071) roll requested 2024-11-22T23:54:57,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697080 2024-11-22T23:54:57,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697080 2024-11-22T23:54:57,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697080) roll requested 2024-11-22T23:54:57,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697087 2024-11-22T23:54:57,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697087 2024-11-22T23:54:57,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697087) roll requested 2024-11-22T23:54:57,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697094 2024-11-22T23:54:57,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697094 2024-11-22T23:54:57,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697094) roll requested 2024-11-22T23:54:57,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697102 2024-11-22T23:54:57,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697102 2024-11-22T23:54:57,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697102) roll requested 2024-11-22T23:54:57,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697109 2024-11-22T23:54:57,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697109 2024-11-22T23:54:57,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697109) roll requested 2024-11-22T23:54:57,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697116 2024-11-22T23:54:57,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697116 2024-11-22T23:54:57,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697116) roll requested 2024-11-22T23:54:57,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697127 2024-11-22T23:54:57,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697127 2024-11-22T23:54:57,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697127) roll requested 2024-11-22T23:54:57,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697135 2024-11-22T23:54:57,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697135 2024-11-22T23:54:57,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697135) roll requested 2024-11-22T23:54:57,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697142 2024-11-22T23:54:57,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697142 2024-11-22T23:54:57,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697142) roll requested 2024-11-22T23:54:57,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697149 2024-11-22T23:54:57,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697149 2024-11-22T23:54:57,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697149) roll requested 2024-11-22T23:54:57,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697157 2024-11-22T23:54:57,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697157 2024-11-22T23:54:57,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697157) roll requested 2024-11-22T23:54:57,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697164 2024-11-22T23:54:57,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697164 2024-11-22T23:54:57,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697164) roll requested 2024-11-22T23:54:57,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697171 2024-11-22T23:54:57,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697171 2024-11-22T23:54:57,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697171) roll requested 2024-11-22T23:54:57,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697179 2024-11-22T23:54:57,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697179 2024-11-22T23:54:57,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697179) roll requested 2024-11-22T23:54:57,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697187 2024-11-22T23:54:57,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697187 2024-11-22T23:54:57,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697187) roll requested 2024-11-22T23:54:57,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697194 2024-11-22T23:54:57,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697194 2024-11-22T23:54:57,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697194) roll requested 2024-11-22T23:54:57,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697201 2024-11-22T23:54:57,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697201 2024-11-22T23:54:57,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697201) roll requested 2024-11-22T23:54:57,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697209 2024-11-22T23:54:57,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697209 2024-11-22T23:54:57,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697209) roll requested 2024-11-22T23:54:57,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697216 2024-11-22T23:54:57,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697216 2024-11-22T23:54:57,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697216) roll requested 2024-11-22T23:54:57,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697223 2024-11-22T23:54:57,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697223 2024-11-22T23:54:57,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697223) roll requested 2024-11-22T23:54:57,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697230 2024-11-22T23:54:57,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697230 2024-11-22T23:54:57,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697230) roll requested 2024-11-22T23:54:57,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697249 2024-11-22T23:54:57,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697249 2024-11-22T23:54:57,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697249) roll requested 2024-11-22T23:54:57,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697258 2024-11-22T23:54:57,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697258 2024-11-22T23:54:57,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697258) roll requested 2024-11-22T23:54:57,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697265 2024-11-22T23:54:57,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697265 2024-11-22T23:54:57,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697265) roll requested 2024-11-22T23:54:57,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697281 2024-11-22T23:54:57,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697281 2024-11-22T23:54:57,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697281) roll requested 2024-11-22T23:54:57,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697292 2024-11-22T23:54:57,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697292 2024-11-22T23:54:57,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697292) roll requested 2024-11-22T23:54:57,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697300 2024-11-22T23:54:57,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697300 2024-11-22T23:54:57,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697300) roll requested 2024-11-22T23:54:57,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697310 2024-11-22T23:54:57,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697310 2024-11-22T23:54:57,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697310) roll requested 2024-11-22T23:54:57,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697317 2024-11-22T23:54:57,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697317 2024-11-22T23:54:57,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697317) roll requested 2024-11-22T23:54:57,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697325 2024-11-22T23:54:57,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697325 2024-11-22T23:54:57,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697325) roll requested 2024-11-22T23:54:57,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697342 2024-11-22T23:54:57,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697342 2024-11-22T23:54:57,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697342) roll requested 2024-11-22T23:54:57,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697350 2024-11-22T23:54:57,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697350 2024-11-22T23:54:57,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697350) roll requested 2024-11-22T23:54:57,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697357 2024-11-22T23:54:57,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697357 2024-11-22T23:54:57,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697357) roll requested 2024-11-22T23:54:57,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697365 2024-11-22T23:54:57,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697365 2024-11-22T23:54:57,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697365) roll requested 2024-11-22T23:54:57,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697372 2024-11-22T23:54:57,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697372 2024-11-22T23:54:57,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697372) roll requested 2024-11-22T23:54:57,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697379 2024-11-22T23:54:57,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697379 2024-11-22T23:54:57,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697379) roll requested 2024-11-22T23:54:57,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697386 2024-11-22T23:54:57,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697386 2024-11-22T23:54:57,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697386) roll requested 2024-11-22T23:54:57,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697393 2024-11-22T23:54:57,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697393 2024-11-22T23:54:57,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697393) roll requested 2024-11-22T23:54:57,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697400 2024-11-22T23:54:57,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697400 2024-11-22T23:54:57,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697400) roll requested 2024-11-22T23:54:57,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697407 2024-11-22T23:54:57,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697407 2024-11-22T23:54:57,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697407) roll requested 2024-11-22T23:54:57,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697414 2024-11-22T23:54:57,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697414 2024-11-22T23:54:57,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697414) roll requested 2024-11-22T23:54:57,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697421 2024-11-22T23:54:57,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697421 2024-11-22T23:54:57,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697421) roll requested 2024-11-22T23:54:57,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697429 2024-11-22T23:54:57,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697429 2024-11-22T23:54:57,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697429) roll requested 2024-11-22T23:54:57,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697435 2024-11-22T23:54:57,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697435 2024-11-22T23:54:57,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697435) roll requested 2024-11-22T23:54:57,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697443 2024-11-22T23:54:57,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697443 2024-11-22T23:54:57,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697443) roll requested 2024-11-22T23:54:57,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697450 2024-11-22T23:54:57,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697450 2024-11-22T23:54:57,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697450) roll requested 2024-11-22T23:54:57,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697457 2024-11-22T23:54:57,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697457 2024-11-22T23:54:57,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697457) roll requested 2024-11-22T23:54:57,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697464 2024-11-22T23:54:57,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697464 2024-11-22T23:54:57,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697464) roll requested 2024-11-22T23:54:57,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697471 2024-11-22T23:54:57,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697471 2024-11-22T23:54:57,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697471) roll requested 2024-11-22T23:54:57,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697478 2024-11-22T23:54:57,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697478 2024-11-22T23:54:57,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697478) roll requested 2024-11-22T23:54:57,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697486 2024-11-22T23:54:57,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697486 2024-11-22T23:54:57,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697486) roll requested 2024-11-22T23:54:57,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697493 2024-11-22T23:54:57,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697493 2024-11-22T23:54:57,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697493) roll requested 2024-11-22T23:54:57,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697500 2024-11-22T23:54:57,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697500 2024-11-22T23:54:57,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697500) roll requested 2024-11-22T23:54:57,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697508 2024-11-22T23:54:57,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697508 2024-11-22T23:54:57,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697508) roll requested 2024-11-22T23:54:57,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697520 2024-11-22T23:54:57,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697520 2024-11-22T23:54:57,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697520) roll requested 2024-11-22T23:54:57,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697527 2024-11-22T23:54:57,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697527 2024-11-22T23:54:57,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697527) roll requested 2024-11-22T23:54:57,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697534 2024-11-22T23:54:57,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697534 2024-11-22T23:54:57,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697534) roll requested 2024-11-22T23:54:57,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697541 2024-11-22T23:54:57,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697541 2024-11-22T23:54:57,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697541) roll requested 2024-11-22T23:54:57,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697549 2024-11-22T23:54:57,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697549 2024-11-22T23:54:57,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697549) roll requested 2024-11-22T23:54:57,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697555 2024-11-22T23:54:57,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697555 2024-11-22T23:54:57,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697555) roll requested 2024-11-22T23:54:57,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697562 2024-11-22T23:54:57,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697562 2024-11-22T23:54:57,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697562) roll requested 2024-11-22T23:54:57,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697570 2024-11-22T23:54:57,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697570 2024-11-22T23:54:57,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697570) roll requested 2024-11-22T23:54:57,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697578 2024-11-22T23:54:57,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697578 2024-11-22T23:54:57,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697578) roll requested 2024-11-22T23:54:57,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697586 2024-11-22T23:54:57,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697586 2024-11-22T23:54:57,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697586) roll requested 2024-11-22T23:54:57,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697593 2024-11-22T23:54:57,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697593 2024-11-22T23:54:57,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697593) roll requested 2024-11-22T23:54:57,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697602 2024-11-22T23:54:57,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697602 2024-11-22T23:54:57,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697602) roll requested 2024-11-22T23:54:57,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697609 2024-11-22T23:54:57,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697609 2024-11-22T23:54:57,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697609) roll requested 2024-11-22T23:54:57,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697617 2024-11-22T23:54:57,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697617 2024-11-22T23:54:57,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697617) roll requested 2024-11-22T23:54:57,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697624 2024-11-22T23:54:57,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697624 2024-11-22T23:54:57,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697624) roll requested 2024-11-22T23:54:57,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697632 2024-11-22T23:54:57,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697632 2024-11-22T23:54:57,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697632) roll requested 2024-11-22T23:54:57,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697640 2024-11-22T23:54:57,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697640 2024-11-22T23:54:57,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697640) roll requested 2024-11-22T23:54:57,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697647 2024-11-22T23:54:57,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697647 2024-11-22T23:54:57,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697647) roll requested 2024-11-22T23:54:57,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697655 2024-11-22T23:54:57,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697655 2024-11-22T23:54:57,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697655) roll requested 2024-11-22T23:54:57,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697661 2024-11-22T23:54:57,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697661 2024-11-22T23:54:57,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697661) roll requested 2024-11-22T23:54:57,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697669 2024-11-22T23:54:57,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697669 2024-11-22T23:54:57,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697669) roll requested 2024-11-22T23:54:57,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697676 2024-11-22T23:54:57,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697676 2024-11-22T23:54:57,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697676) roll requested 2024-11-22T23:54:57,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697683 2024-11-22T23:54:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697683 2024-11-22T23:54:57,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697683) roll requested 2024-11-22T23:54:57,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697690 2024-11-22T23:54:57,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697690 2024-11-22T23:54:57,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697690) roll requested 2024-11-22T23:54:57,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697697 2024-11-22T23:54:57,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697697 2024-11-22T23:54:57,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697697) roll requested 2024-11-22T23:54:57,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697704 2024-11-22T23:54:57,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697704 2024-11-22T23:54:57,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697704) roll requested 2024-11-22T23:54:57,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697711 2024-11-22T23:54:57,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697711 2024-11-22T23:54:57,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697711) roll requested 2024-11-22T23:54:57,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697717 2024-11-22T23:54:57,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697717 2024-11-22T23:54:57,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697717) roll requested 2024-11-22T23:54:57,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697725 2024-11-22T23:54:57,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697725 2024-11-22T23:54:57,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697725) roll requested 2024-11-22T23:54:57,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697733 2024-11-22T23:54:57,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697733 2024-11-22T23:54:57,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697733) roll requested 2024-11-22T23:54:57,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697746 2024-11-22T23:54:57,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697746 2024-11-22T23:54:57,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697746) roll requested 2024-11-22T23:54:57,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697753 2024-11-22T23:54:57,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697753 2024-11-22T23:54:57,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697753) roll requested 2024-11-22T23:54:57,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697760 2024-11-22T23:54:57,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697760 2024-11-22T23:54:57,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697760) roll requested 2024-11-22T23:54:57,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697768 2024-11-22T23:54:57,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697768 2024-11-22T23:54:57,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697768) roll requested 2024-11-22T23:54:57,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697775 2024-11-22T23:54:57,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697775 2024-11-22T23:54:57,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697775) roll requested 2024-11-22T23:54:57,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697783 2024-11-22T23:54:57,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697783 2024-11-22T23:54:57,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697783) roll requested 2024-11-22T23:54:57,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697805 2024-11-22T23:54:57,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697805 2024-11-22T23:54:57,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697805) roll requested 2024-11-22T23:54:57,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697817 2024-11-22T23:54:57,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697817 2024-11-22T23:54:57,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697817) roll requested 2024-11-22T23:54:57,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697837 2024-11-22T23:54:57,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697837 2024-11-22T23:54:57,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697837) roll requested 2024-11-22T23:54:57,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697848 2024-11-22T23:54:57,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697848 2024-11-22T23:54:57,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697848) roll requested 2024-11-22T23:54:57,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697860 2024-11-22T23:54:57,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697860 2024-11-22T23:54:57,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697860) roll requested 2024-11-22T23:54:57,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697868 2024-11-22T23:54:57,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697868 2024-11-22T23:54:57,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697868) roll requested 2024-11-22T23:54:57,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697876 2024-11-22T23:54:57,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697876 2024-11-22T23:54:57,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697876) roll requested 2024-11-22T23:54:57,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697884 2024-11-22T23:54:57,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697884 2024-11-22T23:54:57,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697884) roll requested 2024-11-22T23:54:57,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697891 2024-11-22T23:54:57,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697891 2024-11-22T23:54:57,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697891) roll requested 2024-11-22T23:54:57,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697898 2024-11-22T23:54:57,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697898 2024-11-22T23:54:57,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697898) roll requested 2024-11-22T23:54:57,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697905 2024-11-22T23:54:57,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697905 2024-11-22T23:54:57,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697905) roll requested 2024-11-22T23:54:57,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697913 2024-11-22T23:54:57,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697913 2024-11-22T23:54:57,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697913) roll requested 2024-11-22T23:54:57,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697921 2024-11-22T23:54:57,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697921 2024-11-22T23:54:57,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697921) roll requested 2024-11-22T23:54:57,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697927 2024-11-22T23:54:57,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697927 2024-11-22T23:54:57,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697927) roll requested 2024-11-22T23:54:57,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697935 2024-11-22T23:54:57,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697935 2024-11-22T23:54:57,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697935) roll requested 2024-11-22T23:54:57,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697942 2024-11-22T23:54:57,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697942 2024-11-22T23:54:57,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697942) roll requested 2024-11-22T23:54:57,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697949 2024-11-22T23:54:57,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697949 2024-11-22T23:54:57,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697949) roll requested 2024-11-22T23:54:57,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697957 2024-11-22T23:54:57,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697957 2024-11-22T23:54:57,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697957) roll requested 2024-11-22T23:54:57,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697964 2024-11-22T23:54:57,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697964 2024-11-22T23:54:57,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697964) roll requested 2024-11-22T23:54:57,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697982 2024-11-22T23:54:57,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:57,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697982 2024-11-22T23:54:57,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:57,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:57,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697982) roll requested 2024-11-22T23:54:57,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319697989 2024-11-22T23:54:58,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697989 2024-11-22T23:54:58,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319697989) roll requested 2024-11-22T23:54:58,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698005 2024-11-22T23:54:58,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319697989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698005 2024-11-22T23:54:58,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698005) roll requested 2024-11-22T23:54:58,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698012 2024-11-22T23:54:58,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698012 2024-11-22T23:54:58,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698012) roll requested 2024-11-22T23:54:58,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698019 2024-11-22T23:54:58,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698019 2024-11-22T23:54:58,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698019) roll requested 2024-11-22T23:54:58,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698027 2024-11-22T23:54:58,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698027 2024-11-22T23:54:58,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698027) roll requested 2024-11-22T23:54:58,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698037 2024-11-22T23:54:58,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698037 2024-11-22T23:54:58,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698037) roll requested 2024-11-22T23:54:58,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698044 2024-11-22T23:54:58,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698044 2024-11-22T23:54:58,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698044) roll requested 2024-11-22T23:54:58,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698057 2024-11-22T23:54:58,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698057 2024-11-22T23:54:58,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698057) roll requested 2024-11-22T23:54:58,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698064 2024-11-22T23:54:58,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698064 2024-11-22T23:54:58,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698064) roll requested 2024-11-22T23:54:58,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698072 2024-11-22T23:54:58,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698072 2024-11-22T23:54:58,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698072) roll requested 2024-11-22T23:54:58,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698085 2024-11-22T23:54:58,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698085 2024-11-22T23:54:58,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698085) roll requested 2024-11-22T23:54:58,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698105 2024-11-22T23:54:58,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698105 2024-11-22T23:54:58,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698105) roll requested 2024-11-22T23:54:58,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698114 2024-11-22T23:54:58,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698114 2024-11-22T23:54:58,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698114) roll requested 2024-11-22T23:54:58,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698125 2024-11-22T23:54:58,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698125 2024-11-22T23:54:58,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698125) roll requested 2024-11-22T23:54:58,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698132 2024-11-22T23:54:58,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698132 2024-11-22T23:54:58,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698132) roll requested 2024-11-22T23:54:58,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698139 2024-11-22T23:54:58,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698139 2024-11-22T23:54:58,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698139) roll requested 2024-11-22T23:54:58,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698147 2024-11-22T23:54:58,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698147 2024-11-22T23:54:58,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698147) roll requested 2024-11-22T23:54:58,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698154 2024-11-22T23:54:58,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698154 2024-11-22T23:54:58,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698154) roll requested 2024-11-22T23:54:58,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698161 2024-11-22T23:54:58,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698161 2024-11-22T23:54:58,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698161) roll requested 2024-11-22T23:54:58,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698168 2024-11-22T23:54:58,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698168 2024-11-22T23:54:58,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698168) roll requested 2024-11-22T23:54:58,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698178 2024-11-22T23:54:58,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698178 2024-11-22T23:54:58,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698178) roll requested 2024-11-22T23:54:58,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698185 2024-11-22T23:54:58,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698185 2024-11-22T23:54:58,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698185) roll requested 2024-11-22T23:54:58,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698192 2024-11-22T23:54:58,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698192 2024-11-22T23:54:58,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698192) roll requested 2024-11-22T23:54:58,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698200 2024-11-22T23:54:58,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698200 2024-11-22T23:54:58,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698200) roll requested 2024-11-22T23:54:58,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698208 2024-11-22T23:54:58,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698208 2024-11-22T23:54:58,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698208) roll requested 2024-11-22T23:54:58,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698219 2024-11-22T23:54:58,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698219 2024-11-22T23:54:58,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698219) roll requested 2024-11-22T23:54:58,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698233 2024-11-22T23:54:58,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698233 2024-11-22T23:54:58,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698233) roll requested 2024-11-22T23:54:58,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698242 2024-11-22T23:54:58,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698242 2024-11-22T23:54:58,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698242) roll requested 2024-11-22T23:54:58,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698249 2024-11-22T23:54:58,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698249 2024-11-22T23:54:58,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698249) roll requested 2024-11-22T23:54:58,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698256 2024-11-22T23:54:58,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698256 2024-11-22T23:54:58,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698256) roll requested 2024-11-22T23:54:58,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698263 2024-11-22T23:54:58,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698263 2024-11-22T23:54:58,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698263) roll requested 2024-11-22T23:54:58,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698278 2024-11-22T23:54:58,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698278 2024-11-22T23:54:58,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698278) roll requested 2024-11-22T23:54:58,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698295 2024-11-22T23:54:58,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698295 2024-11-22T23:54:58,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698295) roll requested 2024-11-22T23:54:58,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698305 2024-11-22T23:54:58,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698305 2024-11-22T23:54:58,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698305) roll requested 2024-11-22T23:54:58,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698312 2024-11-22T23:54:58,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698312 2024-11-22T23:54:58,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698312) roll requested 2024-11-22T23:54:58,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698320 2024-11-22T23:54:58,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698320 2024-11-22T23:54:58,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698320) roll requested 2024-11-22T23:54:58,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698327 2024-11-22T23:54:58,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698327 2024-11-22T23:54:58,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698327) roll requested 2024-11-22T23:54:58,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698334 2024-11-22T23:54:58,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698334 2024-11-22T23:54:58,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698334) roll requested 2024-11-22T23:54:58,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698346 2024-11-22T23:54:58,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698346 2024-11-22T23:54:58,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698346) roll requested 2024-11-22T23:54:58,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698353 2024-11-22T23:54:58,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698353 2024-11-22T23:54:58,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698353) roll requested 2024-11-22T23:54:58,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698360 2024-11-22T23:54:58,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698360 2024-11-22T23:54:58,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698360) roll requested 2024-11-22T23:54:58,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698369 2024-11-22T23:54:58,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698369 2024-11-22T23:54:58,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698369) roll requested 2024-11-22T23:54:58,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698376 2024-11-22T23:54:58,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698376 2024-11-22T23:54:58,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698376) roll requested 2024-11-22T23:54:58,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698395 2024-11-22T23:54:58,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698395 2024-11-22T23:54:58,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698395) roll requested 2024-11-22T23:54:58,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698403 2024-11-22T23:54:58,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698403 2024-11-22T23:54:58,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698403) roll requested 2024-11-22T23:54:58,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698411 2024-11-22T23:54:58,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698411 2024-11-22T23:54:58,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698411) roll requested 2024-11-22T23:54:58,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698417 2024-11-22T23:54:58,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698417 2024-11-22T23:54:58,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698417) roll requested 2024-11-22T23:54:58,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698424 2024-11-22T23:54:58,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698424 2024-11-22T23:54:58,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698424) roll requested 2024-11-22T23:54:58,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698431 2024-11-22T23:54:58,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698431 2024-11-22T23:54:58,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698431) roll requested 2024-11-22T23:54:58,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698438 2024-11-22T23:54:58,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698438 2024-11-22T23:54:58,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698438) roll requested 2024-11-22T23:54:58,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698444 2024-11-22T23:54:58,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698444 2024-11-22T23:54:58,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698444) roll requested 2024-11-22T23:54:58,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698451 2024-11-22T23:54:58,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698451 2024-11-22T23:54:58,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698451) roll requested 2024-11-22T23:54:58,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698459 2024-11-22T23:54:58,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698459 2024-11-22T23:54:58,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698459) roll requested 2024-11-22T23:54:58,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698466 2024-11-22T23:54:58,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698466 2024-11-22T23:54:58,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698466) roll requested 2024-11-22T23:54:58,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698473 2024-11-22T23:54:58,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698473 2024-11-22T23:54:58,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698473) roll requested 2024-11-22T23:54:58,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698481 2024-11-22T23:54:58,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698481 2024-11-22T23:54:58,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698481) roll requested 2024-11-22T23:54:58,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698487 2024-11-22T23:54:58,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698487 2024-11-22T23:54:58,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698487) roll requested 2024-11-22T23:54:58,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698494 2024-11-22T23:54:58,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698494 2024-11-22T23:54:58,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698494) roll requested 2024-11-22T23:54:58,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698502 2024-11-22T23:54:58,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698502 2024-11-22T23:54:58,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698502) roll requested 2024-11-22T23:54:58,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698508 2024-11-22T23:54:58,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698508 2024-11-22T23:54:58,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698508) roll requested 2024-11-22T23:54:58,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698515 2024-11-22T23:54:58,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698515 2024-11-22T23:54:58,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698515) roll requested 2024-11-22T23:54:58,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698522 2024-11-22T23:54:58,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698522 2024-11-22T23:54:58,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698522) roll requested 2024-11-22T23:54:58,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698529 2024-11-22T23:54:58,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698529 2024-11-22T23:54:58,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698529) roll requested 2024-11-22T23:54:58,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698536 2024-11-22T23:54:58,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698536 2024-11-22T23:54:58,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698536) roll requested 2024-11-22T23:54:58,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698543 2024-11-22T23:54:58,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698543 2024-11-22T23:54:58,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698543) roll requested 2024-11-22T23:54:58,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698550 2024-11-22T23:54:58,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698550 2024-11-22T23:54:58,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698550) roll requested 2024-11-22T23:54:58,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698565 2024-11-22T23:54:58,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698565 2024-11-22T23:54:58,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698565) roll requested 2024-11-22T23:54:58,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698577 2024-11-22T23:54:58,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698577 2024-11-22T23:54:58,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698577) roll requested 2024-11-22T23:54:58,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698584 2024-11-22T23:54:58,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698584 2024-11-22T23:54:58,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698584) roll requested 2024-11-22T23:54:58,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698590 2024-11-22T23:54:58,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698590 2024-11-22T23:54:58,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698590) roll requested 2024-11-22T23:54:58,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698597 2024-11-22T23:54:58,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698597 2024-11-22T23:54:58,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698597) roll requested 2024-11-22T23:54:58,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698604 2024-11-22T23:54:58,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698604 2024-11-22T23:54:58,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698604) roll requested 2024-11-22T23:54:58,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698611 2024-11-22T23:54:58,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698611 2024-11-22T23:54:58,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698611) roll requested 2024-11-22T23:54:58,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698619 2024-11-22T23:54:58,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698619 2024-11-22T23:54:58,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698619) roll requested 2024-11-22T23:54:58,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698625 2024-11-22T23:54:58,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698625 2024-11-22T23:54:58,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698625) roll requested 2024-11-22T23:54:58,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698632 2024-11-22T23:54:58,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698632 2024-11-22T23:54:58,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698632) roll requested 2024-11-22T23:54:58,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698640 2024-11-22T23:54:58,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698640 2024-11-22T23:54:58,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698640) roll requested 2024-11-22T23:54:58,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698647 2024-11-22T23:54:58,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698647 2024-11-22T23:54:58,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698647) roll requested 2024-11-22T23:54:58,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698654 2024-11-22T23:54:58,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698654 2024-11-22T23:54:58,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698654) roll requested 2024-11-22T23:54:58,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698661 2024-11-22T23:54:58,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698661 2024-11-22T23:54:58,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698661) roll requested 2024-11-22T23:54:58,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698668 2024-11-22T23:54:58,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698668 2024-11-22T23:54:58,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698668) roll requested 2024-11-22T23:54:58,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698683 2024-11-22T23:54:58,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698683 2024-11-22T23:54:58,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698683) roll requested 2024-11-22T23:54:58,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698693 2024-11-22T23:54:58,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698693 2024-11-22T23:54:58,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698693) roll requested 2024-11-22T23:54:58,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698705 2024-11-22T23:54:58,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698705 2024-11-22T23:54:58,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698705) roll requested 2024-11-22T23:54:58,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698712 2024-11-22T23:54:58,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698712 2024-11-22T23:54:58,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698712) roll requested 2024-11-22T23:54:58,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698718 2024-11-22T23:54:58,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698718 2024-11-22T23:54:58,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698718) roll requested 2024-11-22T23:54:58,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698725 2024-11-22T23:54:58,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698725 2024-11-22T23:54:58,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698725) roll requested 2024-11-22T23:54:58,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698732 2024-11-22T23:54:58,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698732 2024-11-22T23:54:58,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698732) roll requested 2024-11-22T23:54:58,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698739 2024-11-22T23:54:58,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698739 2024-11-22T23:54:58,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698739) roll requested 2024-11-22T23:54:58,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698746 2024-11-22T23:54:58,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698746 2024-11-22T23:54:58,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698746) roll requested 2024-11-22T23:54:58,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698753 2024-11-22T23:54:58,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698753 2024-11-22T23:54:58,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698753) roll requested 2024-11-22T23:54:58,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698760 2024-11-22T23:54:58,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698760 2024-11-22T23:54:58,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698760) roll requested 2024-11-22T23:54:58,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698767 2024-11-22T23:54:58,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698767 2024-11-22T23:54:58,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698767) roll requested 2024-11-22T23:54:58,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698774 2024-11-22T23:54:58,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698774 2024-11-22T23:54:58,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698774) roll requested 2024-11-22T23:54:58,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698780 2024-11-22T23:54:58,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698780 2024-11-22T23:54:58,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698780) roll requested 2024-11-22T23:54:58,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698792 2024-11-22T23:54:58,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698792 2024-11-22T23:54:58,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698792) roll requested 2024-11-22T23:54:58,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698799 2024-11-22T23:54:58,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698799 2024-11-22T23:54:58,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698799) roll requested 2024-11-22T23:54:58,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698806 2024-11-22T23:54:58,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698806 2024-11-22T23:54:58,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698806) roll requested 2024-11-22T23:54:58,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698813 2024-11-22T23:54:58,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698813 2024-11-22T23:54:58,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698813) roll requested 2024-11-22T23:54:58,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698820 2024-11-22T23:54:58,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698820 2024-11-22T23:54:58,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698820) roll requested 2024-11-22T23:54:58,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698827 2024-11-22T23:54:58,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698827 2024-11-22T23:54:58,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698827) roll requested 2024-11-22T23:54:58,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698833 2024-11-22T23:54:58,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698833 2024-11-22T23:54:58,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698833) roll requested 2024-11-22T23:54:58,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698841 2024-11-22T23:54:58,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698841 2024-11-22T23:54:58,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698841) roll requested 2024-11-22T23:54:58,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698849 2024-11-22T23:54:58,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698849 2024-11-22T23:54:58,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698849) roll requested 2024-11-22T23:54:58,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698861 2024-11-22T23:54:58,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698861 2024-11-22T23:54:58,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698861) roll requested 2024-11-22T23:54:58,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698868 2024-11-22T23:54:58,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698868 2024-11-22T23:54:58,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698868) roll requested 2024-11-22T23:54:58,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698875 2024-11-22T23:54:58,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698875 2024-11-22T23:54:58,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698875) roll requested 2024-11-22T23:54:58,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698882 2024-11-22T23:54:58,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698882 2024-11-22T23:54:58,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698882) roll requested 2024-11-22T23:54:58,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698889 2024-11-22T23:54:58,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698889 2024-11-22T23:54:58,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698889) roll requested 2024-11-22T23:54:58,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698895 2024-11-22T23:54:58,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698895 2024-11-22T23:54:58,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698895) roll requested 2024-11-22T23:54:58,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698902 2024-11-22T23:54:58,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698902 2024-11-22T23:54:58,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698902) roll requested 2024-11-22T23:54:58,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698908 2024-11-22T23:54:58,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698908 2024-11-22T23:54:58,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698908) roll requested 2024-11-22T23:54:58,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698915 2024-11-22T23:54:58,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698915 2024-11-22T23:54:58,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698915) roll requested 2024-11-22T23:54:58,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698922 2024-11-22T23:54:58,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698922 2024-11-22T23:54:58,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698922) roll requested 2024-11-22T23:54:58,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698929 2024-11-22T23:54:58,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698929 2024-11-22T23:54:58,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698929) roll requested 2024-11-22T23:54:58,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698937 2024-11-22T23:54:58,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698937 2024-11-22T23:54:58,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698937) roll requested 2024-11-22T23:54:58,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698944 2024-11-22T23:54:58,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698944 2024-11-22T23:54:58,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698944) roll requested 2024-11-22T23:54:58,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698951 2024-11-22T23:54:58,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698951 2024-11-22T23:54:58,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698951) roll requested 2024-11-22T23:54:58,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698958 2024-11-22T23:54:58,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698958 2024-11-22T23:54:58,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698958) roll requested 2024-11-22T23:54:58,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698965 2024-11-22T23:54:58,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698965 2024-11-22T23:54:58,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698965) roll requested 2024-11-22T23:54:58,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698972 2024-11-22T23:54:58,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698972 2024-11-22T23:54:58,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698972) roll requested 2024-11-22T23:54:58,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698979 2024-11-22T23:54:58,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698979 2024-11-22T23:54:58,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698979) roll requested 2024-11-22T23:54:58,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698986 2024-11-22T23:54:58,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698986 2024-11-22T23:54:58,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:58,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:58,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698986) roll requested 2024-11-22T23:54:58,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319698993 2024-11-22T23:54:58,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:58,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698993 2024-11-22T23:54:59,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319698993) roll requested 2024-11-22T23:54:59,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699000 2024-11-22T23:54:59,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319698993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699000 2024-11-22T23:54:59,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699000) roll requested 2024-11-22T23:54:59,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699007 2024-11-22T23:54:59,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699007 2024-11-22T23:54:59,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699007) roll requested 2024-11-22T23:54:59,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699014 2024-11-22T23:54:59,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699014 2024-11-22T23:54:59,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699014) roll requested 2024-11-22T23:54:59,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699022 2024-11-22T23:54:59,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699022 2024-11-22T23:54:59,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699022) roll requested 2024-11-22T23:54:59,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699029 2024-11-22T23:54:59,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699029 2024-11-22T23:54:59,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699029) roll requested 2024-11-22T23:54:59,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699043 2024-11-22T23:54:59,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699043 2024-11-22T23:54:59,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699043) roll requested 2024-11-22T23:54:59,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699054 2024-11-22T23:54:59,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699054 2024-11-22T23:54:59,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699054) roll requested 2024-11-22T23:54:59,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699066 2024-11-22T23:54:59,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699066 2024-11-22T23:54:59,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699066) roll requested 2024-11-22T23:54:59,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699074 2024-11-22T23:54:59,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699074 2024-11-22T23:54:59,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699074) roll requested 2024-11-22T23:54:59,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699095 2024-11-22T23:54:59,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699095 2024-11-22T23:54:59,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699095) roll requested 2024-11-22T23:54:59,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699102 2024-11-22T23:54:59,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699102 2024-11-22T23:54:59,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699102) roll requested 2024-11-22T23:54:59,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699111 2024-11-22T23:54:59,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699111 2024-11-22T23:54:59,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699111) roll requested 2024-11-22T23:54:59,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699118 2024-11-22T23:54:59,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699118 2024-11-22T23:54:59,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699118) roll requested 2024-11-22T23:54:59,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699125 2024-11-22T23:54:59,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699125 2024-11-22T23:54:59,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699125) roll requested 2024-11-22T23:54:59,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699132 2024-11-22T23:54:59,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699132 2024-11-22T23:54:59,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699132) roll requested 2024-11-22T23:54:59,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699139 2024-11-22T23:54:59,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699139 2024-11-22T23:54:59,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699139) roll requested 2024-11-22T23:54:59,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699146 2024-11-22T23:54:59,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699146 2024-11-22T23:54:59,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699146) roll requested 2024-11-22T23:54:59,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699153 2024-11-22T23:54:59,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699153 2024-11-22T23:54:59,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699153) roll requested 2024-11-22T23:54:59,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699160 2024-11-22T23:54:59,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699160 2024-11-22T23:54:59,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699160) roll requested 2024-11-22T23:54:59,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699167 2024-11-22T23:54:59,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699167 2024-11-22T23:54:59,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699167) roll requested 2024-11-22T23:54:59,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699174 2024-11-22T23:54:59,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699174 2024-11-22T23:54:59,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699174) roll requested 2024-11-22T23:54:59,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699181 2024-11-22T23:54:59,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699181 2024-11-22T23:54:59,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699181) roll requested 2024-11-22T23:54:59,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699187 2024-11-22T23:54:59,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699187 2024-11-22T23:54:59,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699187) roll requested 2024-11-22T23:54:59,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699195 2024-11-22T23:54:59,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699195 2024-11-22T23:54:59,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699195) roll requested 2024-11-22T23:54:59,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699201 2024-11-22T23:54:59,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699201 2024-11-22T23:54:59,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699201) roll requested 2024-11-22T23:54:59,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699208 2024-11-22T23:54:59,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699208 2024-11-22T23:54:59,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699208) roll requested 2024-11-22T23:54:59,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699215 2024-11-22T23:54:59,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699215 2024-11-22T23:54:59,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699215) roll requested 2024-11-22T23:54:59,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699222 2024-11-22T23:54:59,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699222 2024-11-22T23:54:59,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699222) roll requested 2024-11-22T23:54:59,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699229 2024-11-22T23:54:59,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699229 2024-11-22T23:54:59,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699229) roll requested 2024-11-22T23:54:59,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699236 2024-11-22T23:54:59,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699236 2024-11-22T23:54:59,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699236) roll requested 2024-11-22T23:54:59,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699243 2024-11-22T23:54:59,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699243 2024-11-22T23:54:59,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699243) roll requested 2024-11-22T23:54:59,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699250 2024-11-22T23:54:59,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699243 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699250 2024-11-22T23:54:59,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699250) roll requested 2024-11-22T23:54:59,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699257 2024-11-22T23:54:59,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699257 2024-11-22T23:54:59,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699257) roll requested 2024-11-22T23:54:59,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699263 2024-11-22T23:54:59,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699263 2024-11-22T23:54:59,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699263) roll requested 2024-11-22T23:54:59,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699271 2024-11-22T23:54:59,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699271 2024-11-22T23:54:59,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699271) roll requested 2024-11-22T23:54:59,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699278 2024-11-22T23:54:59,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699278 2024-11-22T23:54:59,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699278) roll requested 2024-11-22T23:54:59,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699288 2024-11-22T23:54:59,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699288 2024-11-22T23:54:59,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699288) roll requested 2024-11-22T23:54:59,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699295 2024-11-22T23:54:59,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699295 2024-11-22T23:54:59,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699295) roll requested 2024-11-22T23:54:59,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699302 2024-11-22T23:54:59,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699302 2024-11-22T23:54:59,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699302) roll requested 2024-11-22T23:54:59,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699309 2024-11-22T23:54:59,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699309 2024-11-22T23:54:59,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699309) roll requested 2024-11-22T23:54:59,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699317 2024-11-22T23:54:59,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699317 2024-11-22T23:54:59,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699317) roll requested 2024-11-22T23:54:59,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699327 2024-11-22T23:54:59,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699327 2024-11-22T23:54:59,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699327) roll requested 2024-11-22T23:54:59,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699334 2024-11-22T23:54:59,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699334 2024-11-22T23:54:59,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699334) roll requested 2024-11-22T23:54:59,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699341 2024-11-22T23:54:59,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699341 2024-11-22T23:54:59,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699341) roll requested 2024-11-22T23:54:59,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699348 2024-11-22T23:54:59,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699348 2024-11-22T23:54:59,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699348) roll requested 2024-11-22T23:54:59,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699355 2024-11-22T23:54:59,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699355 2024-11-22T23:54:59,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699355) roll requested 2024-11-22T23:54:59,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699362 2024-11-22T23:54:59,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699362 2024-11-22T23:54:59,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699362) roll requested 2024-11-22T23:54:59,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699372 2024-11-22T23:54:59,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699372 2024-11-22T23:54:59,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699372) roll requested 2024-11-22T23:54:59,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699379 2024-11-22T23:54:59,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699379 2024-11-22T23:54:59,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699379) roll requested 2024-11-22T23:54:59,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699385 2024-11-22T23:54:59,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699385 2024-11-22T23:54:59,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699385) roll requested 2024-11-22T23:54:59,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699392 2024-11-22T23:54:59,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699392 2024-11-22T23:54:59,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699392) roll requested 2024-11-22T23:54:59,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699399 2024-11-22T23:54:59,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699399 2024-11-22T23:54:59,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699399) roll requested 2024-11-22T23:54:59,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699417 2024-11-22T23:54:59,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699417 2024-11-22T23:54:59,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699417) roll requested 2024-11-22T23:54:59,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699428 2024-11-22T23:54:59,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699428 2024-11-22T23:54:59,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699428) roll requested 2024-11-22T23:54:59,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699437 2024-11-22T23:54:59,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699437 2024-11-22T23:54:59,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699437) roll requested 2024-11-22T23:54:59,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699446 2024-11-22T23:54:59,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699446 2024-11-22T23:54:59,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699446) roll requested 2024-11-22T23:54:59,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699453 2024-11-22T23:54:59,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699453 2024-11-22T23:54:59,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699453) roll requested 2024-11-22T23:54:59,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699460 2024-11-22T23:54:59,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699460 2024-11-22T23:54:59,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699460) roll requested 2024-11-22T23:54:59,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699467 2024-11-22T23:54:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699467 2024-11-22T23:54:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699467) roll requested 2024-11-22T23:54:59,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699474 2024-11-22T23:54:59,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699474 2024-11-22T23:54:59,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699474) roll requested 2024-11-22T23:54:59,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699481 2024-11-22T23:54:59,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699481 2024-11-22T23:54:59,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699481) roll requested 2024-11-22T23:54:59,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699489 2024-11-22T23:54:59,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699489 2024-11-22T23:54:59,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699489) roll requested 2024-11-22T23:54:59,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699496 2024-11-22T23:54:59,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699496 2024-11-22T23:54:59,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699496) roll requested 2024-11-22T23:54:59,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699503 2024-11-22T23:54:59,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699503 2024-11-22T23:54:59,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699503) roll requested 2024-11-22T23:54:59,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699510 2024-11-22T23:54:59,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699510 2024-11-22T23:54:59,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699510) roll requested 2024-11-22T23:54:59,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699521 2024-11-22T23:54:59,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699521 2024-11-22T23:54:59,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699521) roll requested 2024-11-22T23:54:59,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699528 2024-11-22T23:54:59,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699528 2024-11-22T23:54:59,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699528) roll requested 2024-11-22T23:54:59,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699535 2024-11-22T23:54:59,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699535 2024-11-22T23:54:59,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699535) roll requested 2024-11-22T23:54:59,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699543 2024-11-22T23:54:59,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699543 2024-11-22T23:54:59,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699543) roll requested 2024-11-22T23:54:59,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699569 2024-11-22T23:54:59,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699569 2024-11-22T23:54:59,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699569) roll requested 2024-11-22T23:54:59,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699576 2024-11-22T23:54:59,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699576 2024-11-22T23:54:59,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699576) roll requested 2024-11-22T23:54:59,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699583 2024-11-22T23:54:59,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699583 2024-11-22T23:54:59,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699583) roll requested 2024-11-22T23:54:59,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699590 2024-11-22T23:54:59,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699590 2024-11-22T23:54:59,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699590) roll requested 2024-11-22T23:54:59,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699598 2024-11-22T23:54:59,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699598 2024-11-22T23:54:59,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699598) roll requested 2024-11-22T23:54:59,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699605 2024-11-22T23:54:59,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699605 2024-11-22T23:54:59,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699605) roll requested 2024-11-22T23:54:59,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699612 2024-11-22T23:54:59,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699612 2024-11-22T23:54:59,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699612) roll requested 2024-11-22T23:54:59,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699618 2024-11-22T23:54:59,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699618 2024-11-22T23:54:59,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699618) roll requested 2024-11-22T23:54:59,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699626 2024-11-22T23:54:59,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699626 2024-11-22T23:54:59,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699626) roll requested 2024-11-22T23:54:59,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699633 2024-11-22T23:54:59,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699633 2024-11-22T23:54:59,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699633) roll requested 2024-11-22T23:54:59,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699640 2024-11-22T23:54:59,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699640 2024-11-22T23:54:59,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699640) roll requested 2024-11-22T23:54:59,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699647 2024-11-22T23:54:59,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699647 2024-11-22T23:54:59,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699647) roll requested 2024-11-22T23:54:59,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699653 2024-11-22T23:54:59,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699653 2024-11-22T23:54:59,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699653) roll requested 2024-11-22T23:54:59,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699660 2024-11-22T23:54:59,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699660 2024-11-22T23:54:59,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699660) roll requested 2024-11-22T23:54:59,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699667 2024-11-22T23:54:59,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699667 2024-11-22T23:54:59,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699667) roll requested 2024-11-22T23:54:59,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699673 2024-11-22T23:54:59,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699673 2024-11-22T23:54:59,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699673) roll requested 2024-11-22T23:54:59,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699680 2024-11-22T23:54:59,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699680 2024-11-22T23:54:59,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699680) roll requested 2024-11-22T23:54:59,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699687 2024-11-22T23:54:59,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699687 2024-11-22T23:54:59,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699687) roll requested 2024-11-22T23:54:59,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699694 2024-11-22T23:54:59,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699694 2024-11-22T23:54:59,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699694) roll requested 2024-11-22T23:54:59,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699701 2024-11-22T23:54:59,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699701 2024-11-22T23:54:59,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699701) roll requested 2024-11-22T23:54:59,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699708 2024-11-22T23:54:59,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699708 2024-11-22T23:54:59,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699708) roll requested 2024-11-22T23:54:59,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699714 2024-11-22T23:54:59,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699714 2024-11-22T23:54:59,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699714) roll requested 2024-11-22T23:54:59,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699721 2024-11-22T23:54:59,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699721 2024-11-22T23:54:59,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699721) roll requested 2024-11-22T23:54:59,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699727 2024-11-22T23:54:59,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699727 2024-11-22T23:54:59,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699727) roll requested 2024-11-22T23:54:59,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699734 2024-11-22T23:54:59,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699734 2024-11-22T23:54:59,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699734) roll requested 2024-11-22T23:54:59,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699741 2024-11-22T23:54:59,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699741 2024-11-22T23:54:59,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699741) roll requested 2024-11-22T23:54:59,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699749 2024-11-22T23:54:59,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699749 2024-11-22T23:54:59,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699749) roll requested 2024-11-22T23:54:59,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699756 2024-11-22T23:54:59,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699756 2024-11-22T23:54:59,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699756) roll requested 2024-11-22T23:54:59,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699763 2024-11-22T23:54:59,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699763 2024-11-22T23:54:59,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699763) roll requested 2024-11-22T23:54:59,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699770 2024-11-22T23:54:59,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699770 2024-11-22T23:54:59,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699770) roll requested 2024-11-22T23:54:59,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699777 2024-11-22T23:54:59,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699777 2024-11-22T23:54:59,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699777) roll requested 2024-11-22T23:54:59,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699783 2024-11-22T23:54:59,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699783 2024-11-22T23:54:59,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699783) roll requested 2024-11-22T23:54:59,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699795 2024-11-22T23:54:59,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699795 2024-11-22T23:54:59,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699795) roll requested 2024-11-22T23:54:59,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699802 2024-11-22T23:54:59,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699802 2024-11-22T23:54:59,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699802) roll requested 2024-11-22T23:54:59,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699808 2024-11-22T23:54:59,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699808 2024-11-22T23:54:59,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699808) roll requested 2024-11-22T23:54:59,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699814 2024-11-22T23:54:59,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699814 2024-11-22T23:54:59,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699814) roll requested 2024-11-22T23:54:59,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699825 2024-11-22T23:54:59,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699825 2024-11-22T23:54:59,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699825) roll requested 2024-11-22T23:54:59,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699833 2024-11-22T23:54:59,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699833 2024-11-22T23:54:59,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699833) roll requested 2024-11-22T23:54:59,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699840 2024-11-22T23:54:59,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699840 2024-11-22T23:54:59,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699840) roll requested 2024-11-22T23:54:59,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699847 2024-11-22T23:54:59,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699847 2024-11-22T23:54:59,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699847) roll requested 2024-11-22T23:54:59,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699854 2024-11-22T23:54:59,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699854 2024-11-22T23:54:59,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699854) roll requested 2024-11-22T23:54:59,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699860 2024-11-22T23:54:59,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699860 2024-11-22T23:54:59,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699860) roll requested 2024-11-22T23:54:59,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699867 2024-11-22T23:54:59,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699867 2024-11-22T23:54:59,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699867) roll requested 2024-11-22T23:54:59,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699874 2024-11-22T23:54:59,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699874 2024-11-22T23:54:59,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699874) roll requested 2024-11-22T23:54:59,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699881 2024-11-22T23:54:59,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699881 2024-11-22T23:54:59,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699881) roll requested 2024-11-22T23:54:59,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699888 2024-11-22T23:54:59,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699888 2024-11-22T23:54:59,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699888) roll requested 2024-11-22T23:54:59,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699894 2024-11-22T23:54:59,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699894 2024-11-22T23:54:59,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699894) roll requested 2024-11-22T23:54:59,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699901 2024-11-22T23:54:59,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699901 2024-11-22T23:54:59,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699901) roll requested 2024-11-22T23:54:59,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699908 2024-11-22T23:54:59,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699908 2024-11-22T23:54:59,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699908) roll requested 2024-11-22T23:54:59,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699915 2024-11-22T23:54:59,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699915 2024-11-22T23:54:59,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699915) roll requested 2024-11-22T23:54:59,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699921 2024-11-22T23:54:59,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699921 2024-11-22T23:54:59,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699921) roll requested 2024-11-22T23:54:59,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699928 2024-11-22T23:54:59,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699928 2024-11-22T23:54:59,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699928) roll requested 2024-11-22T23:54:59,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699937 2024-11-22T23:54:59,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699937 2024-11-22T23:54:59,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699937) roll requested 2024-11-22T23:54:59,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699944 2024-11-22T23:54:59,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699944 2024-11-22T23:54:59,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699944) roll requested 2024-11-22T23:54:59,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699951 2024-11-22T23:54:59,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699951 2024-11-22T23:54:59,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699951) roll requested 2024-11-22T23:54:59,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699957 2024-11-22T23:54:59,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699957 2024-11-22T23:54:59,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699957) roll requested 2024-11-22T23:54:59,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699964 2024-11-22T23:54:59,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699964 2024-11-22T23:54:59,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699964) roll requested 2024-11-22T23:54:59,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699971 2024-11-22T23:54:59,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699971 2024-11-22T23:54:59,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699971) roll requested 2024-11-22T23:54:59,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699978 2024-11-22T23:54:59,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699978 2024-11-22T23:54:59,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699978) roll requested 2024-11-22T23:54:59,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699984 2024-11-22T23:54:59,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699984 2024-11-22T23:54:59,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699984) roll requested 2024-11-22T23:54:59,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699991 2024-11-22T23:54:59,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:54:59,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699991 2024-11-22T23:54:59,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:54:59,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:54:59,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699991) roll requested 2024-11-22T23:54:59,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319699998 2024-11-22T23:55:00,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699998 2024-11-22T23:55:00,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319699998) roll requested 2024-11-22T23:55:00,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700005 2024-11-22T23:55:00,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319699998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700005 2024-11-22T23:55:00,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700005) roll requested 2024-11-22T23:55:00,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700012 2024-11-22T23:55:00,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700012 2024-11-22T23:55:00,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700012) roll requested 2024-11-22T23:55:00,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700019 2024-11-22T23:55:00,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700019 2024-11-22T23:55:00,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700019) roll requested 2024-11-22T23:55:00,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700026 2024-11-22T23:55:00,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700026 2024-11-22T23:55:00,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700026) roll requested 2024-11-22T23:55:00,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700032 2024-11-22T23:55:00,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700032 2024-11-22T23:55:00,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700032) roll requested 2024-11-22T23:55:00,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700040 2024-11-22T23:55:00,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700040 2024-11-22T23:55:00,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700040) roll requested 2024-11-22T23:55:00,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700047 2024-11-22T23:55:00,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700047 2024-11-22T23:55:00,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700047) roll requested 2024-11-22T23:55:00,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700055 2024-11-22T23:55:00,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700055 2024-11-22T23:55:00,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700055) roll requested 2024-11-22T23:55:00,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700062 2024-11-22T23:55:00,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700062 2024-11-22T23:55:00,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700062) roll requested 2024-11-22T23:55:00,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700069 2024-11-22T23:55:00,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700069 2024-11-22T23:55:00,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700069) roll requested 2024-11-22T23:55:00,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700075 2024-11-22T23:55:00,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700075 2024-11-22T23:55:00,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700075) roll requested 2024-11-22T23:55:00,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700082 2024-11-22T23:55:00,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700082 2024-11-22T23:55:00,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700082) roll requested 2024-11-22T23:55:00,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700089 2024-11-22T23:55:00,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700089 2024-11-22T23:55:00,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700089) roll requested 2024-11-22T23:55:00,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700096 2024-11-22T23:55:00,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700096 2024-11-22T23:55:00,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700096) roll requested 2024-11-22T23:55:00,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700103 2024-11-22T23:55:00,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700103 2024-11-22T23:55:00,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700103) roll requested 2024-11-22T23:55:00,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700110 2024-11-22T23:55:00,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700110 2024-11-22T23:55:00,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700110) roll requested 2024-11-22T23:55:00,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700117 2024-11-22T23:55:00,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700117 2024-11-22T23:55:00,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700117) roll requested 2024-11-22T23:55:00,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700124 2024-11-22T23:55:00,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700124 2024-11-22T23:55:00,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700124) roll requested 2024-11-22T23:55:00,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700131 2024-11-22T23:55:00,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700131 2024-11-22T23:55:00,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700131) roll requested 2024-11-22T23:55:00,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700138 2024-11-22T23:55:00,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700138 2024-11-22T23:55:00,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700138) roll requested 2024-11-22T23:55:00,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700145 2024-11-22T23:55:00,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700145 2024-11-22T23:55:00,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700145) roll requested 2024-11-22T23:55:00,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700152 2024-11-22T23:55:00,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700152 2024-11-22T23:55:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700152) roll requested 2024-11-22T23:55:00,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700158 2024-11-22T23:55:00,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700158 2024-11-22T23:55:00,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700158) roll requested 2024-11-22T23:55:00,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700166 2024-11-22T23:55:00,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700166 2024-11-22T23:55:00,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700166) roll requested 2024-11-22T23:55:00,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700173 2024-11-22T23:55:00,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700173 2024-11-22T23:55:00,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700173) roll requested 2024-11-22T23:55:00,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700179 2024-11-22T23:55:00,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700179 2024-11-22T23:55:00,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700179) roll requested 2024-11-22T23:55:00,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700186 2024-11-22T23:55:00,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700186 2024-11-22T23:55:00,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700186) roll requested 2024-11-22T23:55:00,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700192 2024-11-22T23:55:00,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700192 2024-11-22T23:55:00,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700192) roll requested 2024-11-22T23:55:00,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700202 2024-11-22T23:55:00,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700202 2024-11-22T23:55:00,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700202) roll requested 2024-11-22T23:55:00,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700209 2024-11-22T23:55:00,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700209 2024-11-22T23:55:00,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700209) roll requested 2024-11-22T23:55:00,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700226 2024-11-22T23:55:00,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700226 2024-11-22T23:55:00,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700226) roll requested 2024-11-22T23:55:00,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700233 2024-11-22T23:55:00,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700233 2024-11-22T23:55:00,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700233) roll requested 2024-11-22T23:55:00,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700240 2024-11-22T23:55:00,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700240 2024-11-22T23:55:00,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700240) roll requested 2024-11-22T23:55:00,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700248 2024-11-22T23:55:00,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700248 2024-11-22T23:55:00,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700248) roll requested 2024-11-22T23:55:00,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700256 2024-11-22T23:55:00,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700256 2024-11-22T23:55:00,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700256) roll requested 2024-11-22T23:55:00,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700263 2024-11-22T23:55:00,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700263 2024-11-22T23:55:00,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700263) roll requested 2024-11-22T23:55:00,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700270 2024-11-22T23:55:00,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700270 2024-11-22T23:55:00,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700270) roll requested 2024-11-22T23:55:00,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700277 2024-11-22T23:55:00,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700277 2024-11-22T23:55:00,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700277) roll requested 2024-11-22T23:55:00,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700284 2024-11-22T23:55:00,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700284 2024-11-22T23:55:00,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700284) roll requested 2024-11-22T23:55:00,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700291 2024-11-22T23:55:00,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700291 2024-11-22T23:55:00,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700291) roll requested 2024-11-22T23:55:00,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700298 2024-11-22T23:55:00,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700298 2024-11-22T23:55:00,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700298) roll requested 2024-11-22T23:55:00,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700306 2024-11-22T23:55:00,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700306 2024-11-22T23:55:00,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700306) roll requested 2024-11-22T23:55:00,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700313 2024-11-22T23:55:00,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700313 2024-11-22T23:55:00,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700313) roll requested 2024-11-22T23:55:00,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700320 2024-11-22T23:55:00,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700320 2024-11-22T23:55:00,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700320) roll requested 2024-11-22T23:55:00,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700327 2024-11-22T23:55:00,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700327 2024-11-22T23:55:00,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700327) roll requested 2024-11-22T23:55:00,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700335 2024-11-22T23:55:00,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700335 2024-11-22T23:55:00,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700335) roll requested 2024-11-22T23:55:00,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700342 2024-11-22T23:55:00,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700342 2024-11-22T23:55:00,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700342) roll requested 2024-11-22T23:55:00,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700350 2024-11-22T23:55:00,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700350 2024-11-22T23:55:00,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700350) roll requested 2024-11-22T23:55:00,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700357 2024-11-22T23:55:00,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700357 2024-11-22T23:55:00,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700357) roll requested 2024-11-22T23:55:00,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700364 2024-11-22T23:55:00,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700364 2024-11-22T23:55:00,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700364) roll requested 2024-11-22T23:55:00,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700371 2024-11-22T23:55:00,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700371 2024-11-22T23:55:00,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700371) roll requested 2024-11-22T23:55:00,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700378 2024-11-22T23:55:00,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700378 2024-11-22T23:55:00,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700378) roll requested 2024-11-22T23:55:00,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700385 2024-11-22T23:55:00,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700385 2024-11-22T23:55:00,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700385) roll requested 2024-11-22T23:55:00,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700392 2024-11-22T23:55:00,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700392 2024-11-22T23:55:00,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700392) roll requested 2024-11-22T23:55:00,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700401 2024-11-22T23:55:00,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700401 2024-11-22T23:55:00,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700401) roll requested 2024-11-22T23:55:00,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700408 2024-11-22T23:55:00,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700408 2024-11-22T23:55:00,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700408) roll requested 2024-11-22T23:55:00,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700416 2024-11-22T23:55:00,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700416 2024-11-22T23:55:00,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700416) roll requested 2024-11-22T23:55:00,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700424 2024-11-22T23:55:00,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700424 2024-11-22T23:55:00,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700424) roll requested 2024-11-22T23:55:00,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700433 2024-11-22T23:55:00,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700433 2024-11-22T23:55:00,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700433) roll requested 2024-11-22T23:55:00,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700440 2024-11-22T23:55:00,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700440 2024-11-22T23:55:00,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700440) roll requested 2024-11-22T23:55:00,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700448 2024-11-22T23:55:00,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700448 2024-11-22T23:55:00,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700448) roll requested 2024-11-22T23:55:00,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700455 2024-11-22T23:55:00,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700455 2024-11-22T23:55:00,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700455) roll requested 2024-11-22T23:55:00,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700468 2024-11-22T23:55:00,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700468 2024-11-22T23:55:00,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700468) roll requested 2024-11-22T23:55:00,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700477 2024-11-22T23:55:00,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700477 2024-11-22T23:55:00,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700477) roll requested 2024-11-22T23:55:00,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700483 2024-11-22T23:55:00,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700483 2024-11-22T23:55:00,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700483) roll requested 2024-11-22T23:55:00,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700490 2024-11-22T23:55:00,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700490 2024-11-22T23:55:00,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700490) roll requested 2024-11-22T23:55:00,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700497 2024-11-22T23:55:00,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700497 2024-11-22T23:55:00,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700497) roll requested 2024-11-22T23:55:00,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700504 2024-11-22T23:55:00,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700504 2024-11-22T23:55:00,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700504) roll requested 2024-11-22T23:55:00,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700510 2024-11-22T23:55:00,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700510 2024-11-22T23:55:00,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700510) roll requested 2024-11-22T23:55:00,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700517 2024-11-22T23:55:00,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700517 2024-11-22T23:55:00,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700517) roll requested 2024-11-22T23:55:00,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700524 2024-11-22T23:55:00,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700524 2024-11-22T23:55:00,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700524) roll requested 2024-11-22T23:55:00,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700531 2024-11-22T23:55:00,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700531 2024-11-22T23:55:00,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700531) roll requested 2024-11-22T23:55:00,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700537 2024-11-22T23:55:00,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700537 2024-11-22T23:55:00,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700537) roll requested 2024-11-22T23:55:00,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700544 2024-11-22T23:55:00,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700544 2024-11-22T23:55:00,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700544) roll requested 2024-11-22T23:55:00,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700551 2024-11-22T23:55:00,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700551 2024-11-22T23:55:00,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700551) roll requested 2024-11-22T23:55:00,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700557 2024-11-22T23:55:00,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700557 2024-11-22T23:55:00,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700557) roll requested 2024-11-22T23:55:00,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700564 2024-11-22T23:55:00,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700564 2024-11-22T23:55:00,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700564) roll requested 2024-11-22T23:55:00,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700571 2024-11-22T23:55:00,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700571 2024-11-22T23:55:00,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700571) roll requested 2024-11-22T23:55:00,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700588 2024-11-22T23:55:00,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700588 2024-11-22T23:55:00,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700588) roll requested 2024-11-22T23:55:00,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700594 2024-11-22T23:55:00,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700594 2024-11-22T23:55:00,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700594) roll requested 2024-11-22T23:55:00,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700601 2024-11-22T23:55:00,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700601 2024-11-22T23:55:00,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700601) roll requested 2024-11-22T23:55:00,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700609 2024-11-22T23:55:00,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700609 2024-11-22T23:55:00,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700609) roll requested 2024-11-22T23:55:00,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700616 2024-11-22T23:55:00,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700616 2024-11-22T23:55:00,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700616) roll requested 2024-11-22T23:55:00,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700623 2024-11-22T23:55:00,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700623 2024-11-22T23:55:00,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700623) roll requested 2024-11-22T23:55:00,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700629 2024-11-22T23:55:00,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700629 2024-11-22T23:55:00,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700629) roll requested 2024-11-22T23:55:00,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700636 2024-11-22T23:55:00,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700636 2024-11-22T23:55:00,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700636) roll requested 2024-11-22T23:55:00,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700643 2024-11-22T23:55:00,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700643 2024-11-22T23:55:00,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700643) roll requested 2024-11-22T23:55:00,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700650 2024-11-22T23:55:00,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700650 2024-11-22T23:55:00,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700650) roll requested 2024-11-22T23:55:00,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700657 2024-11-22T23:55:00,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700657 2024-11-22T23:55:00,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700657) roll requested 2024-11-22T23:55:00,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700664 2024-11-22T23:55:00,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700664 2024-11-22T23:55:00,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700664) roll requested 2024-11-22T23:55:00,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700671 2024-11-22T23:55:00,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700671 2024-11-22T23:55:00,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700671) roll requested 2024-11-22T23:55:00,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700678 2024-11-22T23:55:00,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700678 2024-11-22T23:55:00,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700678) roll requested 2024-11-22T23:55:00,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700684 2024-11-22T23:55:00,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700684 2024-11-22T23:55:00,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700684) roll requested 2024-11-22T23:55:00,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700691 2024-11-22T23:55:00,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700691 2024-11-22T23:55:00,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700691) roll requested 2024-11-22T23:55:00,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700697 2024-11-22T23:55:00,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700697 2024-11-22T23:55:00,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700697) roll requested 2024-11-22T23:55:00,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700704 2024-11-22T23:55:00,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700704 2024-11-22T23:55:00,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700704) roll requested 2024-11-22T23:55:00,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700711 2024-11-22T23:55:00,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700711 2024-11-22T23:55:00,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700711) roll requested 2024-11-22T23:55:00,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700718 2024-11-22T23:55:00,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700718 2024-11-22T23:55:00,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700718) roll requested 2024-11-22T23:55:00,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700725 2024-11-22T23:55:00,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700725 2024-11-22T23:55:00,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700725) roll requested 2024-11-22T23:55:00,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700731 2024-11-22T23:55:00,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700731 2024-11-22T23:55:00,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700731) roll requested 2024-11-22T23:55:00,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700738 2024-11-22T23:55:00,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700738 2024-11-22T23:55:00,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700738) roll requested 2024-11-22T23:55:00,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700744 2024-11-22T23:55:00,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700738 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700744 2024-11-22T23:55:00,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700744) roll requested 2024-11-22T23:55:00,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700751 2024-11-22T23:55:00,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700751 2024-11-22T23:55:00,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700751) roll requested 2024-11-22T23:55:00,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700759 2024-11-22T23:55:00,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700759 2024-11-22T23:55:00,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700759) roll requested 2024-11-22T23:55:00,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700765 2024-11-22T23:55:00,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700765 2024-11-22T23:55:00,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700765) roll requested 2024-11-22T23:55:00,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700772 2024-11-22T23:55:00,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700772 2024-11-22T23:55:00,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700772) roll requested 2024-11-22T23:55:00,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700779 2024-11-22T23:55:00,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700779 2024-11-22T23:55:00,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700779) roll requested 2024-11-22T23:55:00,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700785 2024-11-22T23:55:00,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700785 2024-11-22T23:55:00,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700785) roll requested 2024-11-22T23:55:00,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700792 2024-11-22T23:55:00,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700792 2024-11-22T23:55:00,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700792) roll requested 2024-11-22T23:55:00,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700799 2024-11-22T23:55:00,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700799 2024-11-22T23:55:00,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700799) roll requested 2024-11-22T23:55:00,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700805 2024-11-22T23:55:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700805 2024-11-22T23:55:00,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700805) roll requested 2024-11-22T23:55:00,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700812 2024-11-22T23:55:00,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700812 2024-11-22T23:55:00,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700812) roll requested 2024-11-22T23:55:00,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700818 2024-11-22T23:55:00,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700818 2024-11-22T23:55:00,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700818) roll requested 2024-11-22T23:55:00,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700825 2024-11-22T23:55:00,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700825 2024-11-22T23:55:00,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700825) roll requested 2024-11-22T23:55:00,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700832 2024-11-22T23:55:00,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700832 2024-11-22T23:55:00,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700832) roll requested 2024-11-22T23:55:00,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700839 2024-11-22T23:55:00,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700839 2024-11-22T23:55:00,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700839) roll requested 2024-11-22T23:55:00,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700846 2024-11-22T23:55:00,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700846 2024-11-22T23:55:00,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700846) roll requested 2024-11-22T23:55:00,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700853 2024-11-22T23:55:00,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700853 2024-11-22T23:55:00,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700853) roll requested 2024-11-22T23:55:00,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700860 2024-11-22T23:55:00,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700860 2024-11-22T23:55:00,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700860) roll requested 2024-11-22T23:55:00,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700867 2024-11-22T23:55:00,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700867 2024-11-22T23:55:00,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,874 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700867) roll requested 2024-11-22T23:55:00,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700874 2024-11-22T23:55:00,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700874 2024-11-22T23:55:00,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700874) roll requested 2024-11-22T23:55:00,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700881 2024-11-22T23:55:00,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700881 2024-11-22T23:55:00,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700881) roll requested 2024-11-22T23:55:00,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700888 2024-11-22T23:55:00,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700888 2024-11-22T23:55:00,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700888) roll requested 2024-11-22T23:55:00,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700895 2024-11-22T23:55:00,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700895 2024-11-22T23:55:00,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700895) roll requested 2024-11-22T23:55:00,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700902 2024-11-22T23:55:00,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700902 2024-11-22T23:55:00,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700902) roll requested 2024-11-22T23:55:00,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700909 2024-11-22T23:55:00,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700909 2024-11-22T23:55:00,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700909) roll requested 2024-11-22T23:55:00,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700932 2024-11-22T23:55:00,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700932 2024-11-22T23:55:00,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700932) roll requested 2024-11-22T23:55:00,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700939 2024-11-22T23:55:00,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700939 2024-11-22T23:55:00,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700939) roll requested 2024-11-22T23:55:00,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700946 2024-11-22T23:55:00,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700946 2024-11-22T23:55:00,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700946) roll requested 2024-11-22T23:55:00,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700952 2024-11-22T23:55:00,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700952 2024-11-22T23:55:00,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700952) roll requested 2024-11-22T23:55:00,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700960 2024-11-22T23:55:00,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700960 2024-11-22T23:55:00,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700960) roll requested 2024-11-22T23:55:00,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700966 2024-11-22T23:55:00,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700966 2024-11-22T23:55:00,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700966) roll requested 2024-11-22T23:55:00,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700973 2024-11-22T23:55:00,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700973 2024-11-22T23:55:00,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700973) roll requested 2024-11-22T23:55:00,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700980 2024-11-22T23:55:00,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700980 2024-11-22T23:55:00,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700980) roll requested 2024-11-22T23:55:00,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700986 2024-11-22T23:55:00,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:00,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700986 2024-11-22T23:55:00,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:00,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:00,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700986) roll requested 2024-11-22T23:55:00,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319700995 2024-11-22T23:55:01,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700995 2024-11-22T23:55:01,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319700995) roll requested 2024-11-22T23:55:01,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701001 2024-11-22T23:55:01,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319700995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701001 2024-11-22T23:55:01,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701001) roll requested 2024-11-22T23:55:01,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701008 2024-11-22T23:55:01,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701008 2024-11-22T23:55:01,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701008) roll requested 2024-11-22T23:55:01,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701015 2024-11-22T23:55:01,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701015 2024-11-22T23:55:01,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701015) roll requested 2024-11-22T23:55:01,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701021 2024-11-22T23:55:01,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701021 2024-11-22T23:55:01,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701021) roll requested 2024-11-22T23:55:01,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701028 2024-11-22T23:55:01,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701028 2024-11-22T23:55:01,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701028) roll requested 2024-11-22T23:55:01,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701035 2024-11-22T23:55:01,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701035 2024-11-22T23:55:01,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701035) roll requested 2024-11-22T23:55:01,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701042 2024-11-22T23:55:01,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701042 2024-11-22T23:55:01,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701042) roll requested 2024-11-22T23:55:01,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701049 2024-11-22T23:55:01,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701049 2024-11-22T23:55:01,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701049) roll requested 2024-11-22T23:55:01,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701057 2024-11-22T23:55:01,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701057 2024-11-22T23:55:01,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701057) roll requested 2024-11-22T23:55:01,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701073 2024-11-22T23:55:01,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701073 2024-11-22T23:55:01,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701073) roll requested 2024-11-22T23:55:01,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701080 2024-11-22T23:55:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701080 2024-11-22T23:55:01,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701080) roll requested 2024-11-22T23:55:01,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701087 2024-11-22T23:55:01,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701087 2024-11-22T23:55:01,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701087) roll requested 2024-11-22T23:55:01,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701093 2024-11-22T23:55:01,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701093 2024-11-22T23:55:01,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701093) roll requested 2024-11-22T23:55:01,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701100 2024-11-22T23:55:01,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701100 2024-11-22T23:55:01,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701100) roll requested 2024-11-22T23:55:01,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701106 2024-11-22T23:55:01,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701106 2024-11-22T23:55:01,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701106) roll requested 2024-11-22T23:55:01,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701114 2024-11-22T23:55:01,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701114 2024-11-22T23:55:01,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701114) roll requested 2024-11-22T23:55:01,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701121 2024-11-22T23:55:01,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701121 2024-11-22T23:55:01,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701121) roll requested 2024-11-22T23:55:01,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701128 2024-11-22T23:55:01,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701128 2024-11-22T23:55:01,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701128) roll requested 2024-11-22T23:55:01,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701135 2024-11-22T23:55:01,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701135 2024-11-22T23:55:01,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701135) roll requested 2024-11-22T23:55:01,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701146 2024-11-22T23:55:01,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701146 2024-11-22T23:55:01,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701146) roll requested 2024-11-22T23:55:01,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701153 2024-11-22T23:55:01,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701153 2024-11-22T23:55:01,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701153) roll requested 2024-11-22T23:55:01,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701159 2024-11-22T23:55:01,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701159 2024-11-22T23:55:01,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701159) roll requested 2024-11-22T23:55:01,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701166 2024-11-22T23:55:01,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701166 2024-11-22T23:55:01,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701166) roll requested 2024-11-22T23:55:01,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701173 2024-11-22T23:55:01,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701173 2024-11-22T23:55:01,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701173) roll requested 2024-11-22T23:55:01,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701179 2024-11-22T23:55:01,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701179 2024-11-22T23:55:01,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701179) roll requested 2024-11-22T23:55:01,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701186 2024-11-22T23:55:01,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701186 2024-11-22T23:55:01,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701186) roll requested 2024-11-22T23:55:01,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701196 2024-11-22T23:55:01,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701196 2024-11-22T23:55:01,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701196) roll requested 2024-11-22T23:55:01,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701203 2024-11-22T23:55:01,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701203 2024-11-22T23:55:01,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701203) roll requested 2024-11-22T23:55:01,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701210 2024-11-22T23:55:01,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701210 2024-11-22T23:55:01,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701210) roll requested 2024-11-22T23:55:01,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701218 2024-11-22T23:55:01,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701218 2024-11-22T23:55:01,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701218) roll requested 2024-11-22T23:55:01,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701225 2024-11-22T23:55:01,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701225 2024-11-22T23:55:01,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701225) roll requested 2024-11-22T23:55:01,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701231 2024-11-22T23:55:01,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701231 2024-11-22T23:55:01,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701231) roll requested 2024-11-22T23:55:01,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701241 2024-11-22T23:55:01,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701241 2024-11-22T23:55:01,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701241) roll requested 2024-11-22T23:55:01,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701248 2024-11-22T23:55:01,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701248 2024-11-22T23:55:01,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701248) roll requested 2024-11-22T23:55:01,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701258 2024-11-22T23:55:01,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701258 2024-11-22T23:55:01,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701258) roll requested 2024-11-22T23:55:01,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701265 2024-11-22T23:55:01,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701265 2024-11-22T23:55:01,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701265) roll requested 2024-11-22T23:55:01,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701272 2024-11-22T23:55:01,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701272 2024-11-22T23:55:01,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701272) roll requested 2024-11-22T23:55:01,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701279 2024-11-22T23:55:01,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701279 2024-11-22T23:55:01,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701279) roll requested 2024-11-22T23:55:01,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701285 2024-11-22T23:55:01,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701285 2024-11-22T23:55:01,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701285) roll requested 2024-11-22T23:55:01,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701293 2024-11-22T23:55:01,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701293 2024-11-22T23:55:01,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701293) roll requested 2024-11-22T23:55:01,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701305 2024-11-22T23:55:01,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701305 2024-11-22T23:55:01,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701305) roll requested 2024-11-22T23:55:01,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701313 2024-11-22T23:55:01,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701313 2024-11-22T23:55:01,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701313) roll requested 2024-11-22T23:55:01,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701324 2024-11-22T23:55:01,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701324 2024-11-22T23:55:01,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701324) roll requested 2024-11-22T23:55:01,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701332 2024-11-22T23:55:01,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701332 2024-11-22T23:55:01,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701332) roll requested 2024-11-22T23:55:01,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701339 2024-11-22T23:55:01,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701339 2024-11-22T23:55:01,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701339) roll requested 2024-11-22T23:55:01,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701346 2024-11-22T23:55:01,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701346 2024-11-22T23:55:01,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701346) roll requested 2024-11-22T23:55:01,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701353 2024-11-22T23:55:01,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701353 2024-11-22T23:55:01,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701353) roll requested 2024-11-22T23:55:01,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701362 2024-11-22T23:55:01,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701362 2024-11-22T23:55:01,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701362) roll requested 2024-11-22T23:55:01,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701370 2024-11-22T23:55:01,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701370 2024-11-22T23:55:01,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701370) roll requested 2024-11-22T23:55:01,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701377 2024-11-22T23:55:01,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701377 2024-11-22T23:55:01,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701377) roll requested 2024-11-22T23:55:01,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701384 2024-11-22T23:55:01,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701384 2024-11-22T23:55:01,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701384) roll requested 2024-11-22T23:55:01,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701393 2024-11-22T23:55:01,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701393 2024-11-22T23:55:01,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701393) roll requested 2024-11-22T23:55:01,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701400 2024-11-22T23:55:01,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701400 2024-11-22T23:55:01,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701400) roll requested 2024-11-22T23:55:01,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701407 2024-11-22T23:55:01,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701407 2024-11-22T23:55:01,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701407) roll requested 2024-11-22T23:55:01,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701414 2024-11-22T23:55:01,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701414 2024-11-22T23:55:01,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701414) roll requested 2024-11-22T23:55:01,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701421 2024-11-22T23:55:01,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701421 2024-11-22T23:55:01,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701421) roll requested 2024-11-22T23:55:01,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701430 2024-11-22T23:55:01,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701430 2024-11-22T23:55:01,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701430) roll requested 2024-11-22T23:55:01,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701438 2024-11-22T23:55:01,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701438 2024-11-22T23:55:01,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701438) roll requested 2024-11-22T23:55:01,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701445 2024-11-22T23:55:01,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701445 2024-11-22T23:55:01,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701445) roll requested 2024-11-22T23:55:01,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701452 2024-11-22T23:55:01,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701452 2024-11-22T23:55:01,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701452) roll requested 2024-11-22T23:55:01,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701459 2024-11-22T23:55:01,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701459 2024-11-22T23:55:01,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701459) roll requested 2024-11-22T23:55:01,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701466 2024-11-22T23:55:01,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701466 2024-11-22T23:55:01,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701466) roll requested 2024-11-22T23:55:01,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701475 2024-11-22T23:55:01,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701475 2024-11-22T23:55:01,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701475) roll requested 2024-11-22T23:55:01,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701491 2024-11-22T23:55:01,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701491 2024-11-22T23:55:01,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701491) roll requested 2024-11-22T23:55:01,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701498 2024-11-22T23:55:01,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701498 2024-11-22T23:55:01,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701498) roll requested 2024-11-22T23:55:01,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701504 2024-11-22T23:55:01,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701504 2024-11-22T23:55:01,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701504) roll requested 2024-11-22T23:55:01,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701511 2024-11-22T23:55:01,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701511 2024-11-22T23:55:01,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701511) roll requested 2024-11-22T23:55:01,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701518 2024-11-22T23:55:01,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701518 2024-11-22T23:55:01,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701518) roll requested 2024-11-22T23:55:01,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701526 2024-11-22T23:55:01,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701526 2024-11-22T23:55:01,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701526) roll requested 2024-11-22T23:55:01,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701533 2024-11-22T23:55:01,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701533 2024-11-22T23:55:01,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701533) roll requested 2024-11-22T23:55:01,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701540 2024-11-22T23:55:01,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701540 2024-11-22T23:55:01,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701540) roll requested 2024-11-22T23:55:01,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701547 2024-11-22T23:55:01,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701547 2024-11-22T23:55:01,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701547) roll requested 2024-11-22T23:55:01,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701555 2024-11-22T23:55:01,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701555 2024-11-22T23:55:01,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701555) roll requested 2024-11-22T23:55:01,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701562 2024-11-22T23:55:01,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701562 2024-11-22T23:55:01,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701562) roll requested 2024-11-22T23:55:01,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701570 2024-11-22T23:55:01,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701570 2024-11-22T23:55:01,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701570) roll requested 2024-11-22T23:55:01,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701577 2024-11-22T23:55:01,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701577 2024-11-22T23:55:01,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701577) roll requested 2024-11-22T23:55:01,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701584 2024-11-22T23:55:01,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701584 2024-11-22T23:55:01,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701584) roll requested 2024-11-22T23:55:01,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701590 2024-11-22T23:55:01,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701590 2024-11-22T23:55:01,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701590) roll requested 2024-11-22T23:55:01,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701597 2024-11-22T23:55:01,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701597 2024-11-22T23:55:01,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701597) roll requested 2024-11-22T23:55:01,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701604 2024-11-22T23:55:01,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701604 2024-11-22T23:55:01,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701604) roll requested 2024-11-22T23:55:01,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701611 2024-11-22T23:55:01,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701611 2024-11-22T23:55:01,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701611) roll requested 2024-11-22T23:55:01,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701617 2024-11-22T23:55:01,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701617 2024-11-22T23:55:01,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701617) roll requested 2024-11-22T23:55:01,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701632 2024-11-22T23:55:01,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701632 2024-11-22T23:55:01,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701632) roll requested 2024-11-22T23:55:01,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701641 2024-11-22T23:55:01,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701641 2024-11-22T23:55:01,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701641) roll requested 2024-11-22T23:55:01,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701648 2024-11-22T23:55:01,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701648 2024-11-22T23:55:01,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701648) roll requested 2024-11-22T23:55:01,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701655 2024-11-22T23:55:01,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701655 2024-11-22T23:55:01,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701655) roll requested 2024-11-22T23:55:01,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701664 2024-11-22T23:55:01,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701664 2024-11-22T23:55:01,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701664) roll requested 2024-11-22T23:55:01,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701672 2024-11-22T23:55:01,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701672 2024-11-22T23:55:01,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701672) roll requested 2024-11-22T23:55:01,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701679 2024-11-22T23:55:01,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701679 2024-11-22T23:55:01,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701679) roll requested 2024-11-22T23:55:01,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701686 2024-11-22T23:55:01,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701686 2024-11-22T23:55:01,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701686) roll requested 2024-11-22T23:55:01,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701693 2024-11-22T23:55:01,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701693 2024-11-22T23:55:01,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701693) roll requested 2024-11-22T23:55:01,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701700 2024-11-22T23:55:01,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701700 2024-11-22T23:55:01,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701700) roll requested 2024-11-22T23:55:01,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701706 2024-11-22T23:55:01,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701706 2024-11-22T23:55:01,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701706) roll requested 2024-11-22T23:55:01,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701713 2024-11-22T23:55:01,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701713 2024-11-22T23:55:01,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701713) roll requested 2024-11-22T23:55:01,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701720 2024-11-22T23:55:01,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701720 2024-11-22T23:55:01,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701720) roll requested 2024-11-22T23:55:01,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701727 2024-11-22T23:55:01,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701727 2024-11-22T23:55:01,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701727) roll requested 2024-11-22T23:55:01,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701734 2024-11-22T23:55:01,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701734 2024-11-22T23:55:01,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701734) roll requested 2024-11-22T23:55:01,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701741 2024-11-22T23:55:01,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701741 2024-11-22T23:55:01,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701741) roll requested 2024-11-22T23:55:01,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701748 2024-11-22T23:55:01,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701748 2024-11-22T23:55:01,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701748) roll requested 2024-11-22T23:55:01,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701755 2024-11-22T23:55:01,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701755 2024-11-22T23:55:01,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701755) roll requested 2024-11-22T23:55:01,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701762 2024-11-22T23:55:01,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701762 2024-11-22T23:55:01,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701762) roll requested 2024-11-22T23:55:01,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701769 2024-11-22T23:55:01,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701769 2024-11-22T23:55:01,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701769) roll requested 2024-11-22T23:55:01,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701776 2024-11-22T23:55:01,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701776 2024-11-22T23:55:01,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701776) roll requested 2024-11-22T23:55:01,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701782 2024-11-22T23:55:01,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701782 2024-11-22T23:55:01,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701782) roll requested 2024-11-22T23:55:01,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701789 2024-11-22T23:55:01,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701789 2024-11-22T23:55:01,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701789) roll requested 2024-11-22T23:55:01,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701796 2024-11-22T23:55:01,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701796 2024-11-22T23:55:01,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701796) roll requested 2024-11-22T23:55:01,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701803 2024-11-22T23:55:01,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701803 2024-11-22T23:55:01,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701803) roll requested 2024-11-22T23:55:01,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701812 2024-11-22T23:55:01,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701812 2024-11-22T23:55:01,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701812) roll requested 2024-11-22T23:55:01,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701819 2024-11-22T23:55:01,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701819 2024-11-22T23:55:01,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701819) roll requested 2024-11-22T23:55:01,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701827 2024-11-22T23:55:01,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701827 2024-11-22T23:55:01,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701827) roll requested 2024-11-22T23:55:01,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701834 2024-11-22T23:55:01,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701834 2024-11-22T23:55:01,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701834) roll requested 2024-11-22T23:55:01,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701841 2024-11-22T23:55:01,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701841 2024-11-22T23:55:01,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701841) roll requested 2024-11-22T23:55:01,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701848 2024-11-22T23:55:01,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701848 2024-11-22T23:55:01,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701848) roll requested 2024-11-22T23:55:01,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701855 2024-11-22T23:55:01,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701855 2024-11-22T23:55:01,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701855) roll requested 2024-11-22T23:55:01,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701862 2024-11-22T23:55:01,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701862 2024-11-22T23:55:01,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701862) roll requested 2024-11-22T23:55:01,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701869 2024-11-22T23:55:01,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701869 2024-11-22T23:55:01,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701869) roll requested 2024-11-22T23:55:01,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701876 2024-11-22T23:55:01,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701876 2024-11-22T23:55:01,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701876) roll requested 2024-11-22T23:55:01,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701883 2024-11-22T23:55:01,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701883 2024-11-22T23:55:01,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701883) roll requested 2024-11-22T23:55:01,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701889 2024-11-22T23:55:01,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701889 2024-11-22T23:55:01,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701889) roll requested 2024-11-22T23:55:01,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701897 2024-11-22T23:55:01,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701897 2024-11-22T23:55:01,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701897) roll requested 2024-11-22T23:55:01,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701903 2024-11-22T23:55:01,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701903 2024-11-22T23:55:01,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701903) roll requested 2024-11-22T23:55:01,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701910 2024-11-22T23:55:01,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701910 2024-11-22T23:55:01,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701910) roll requested 2024-11-22T23:55:01,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701918 2024-11-22T23:55:01,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701918 2024-11-22T23:55:01,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701918) roll requested 2024-11-22T23:55:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701925 2024-11-22T23:55:01,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701925 2024-11-22T23:55:01,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701925) roll requested 2024-11-22T23:55:01,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701932 2024-11-22T23:55:01,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701932 2024-11-22T23:55:01,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701932) roll requested 2024-11-22T23:55:01,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701939 2024-11-22T23:55:01,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701939 2024-11-22T23:55:01,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701939) roll requested 2024-11-22T23:55:01,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701946 2024-11-22T23:55:01,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701946 2024-11-22T23:55:01,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701946) roll requested 2024-11-22T23:55:01,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701954 2024-11-22T23:55:01,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701954 2024-11-22T23:55:01,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701954) roll requested 2024-11-22T23:55:01,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701966 2024-11-22T23:55:01,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701966 2024-11-22T23:55:01,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701966) roll requested 2024-11-22T23:55:01,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701973 2024-11-22T23:55:01,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701973 2024-11-22T23:55:01,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701973) roll requested 2024-11-22T23:55:01,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701980 2024-11-22T23:55:01,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701980 2024-11-22T23:55:01,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701980) roll requested 2024-11-22T23:55:01,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701988 2024-11-22T23:55:01,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:01,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701988 2024-11-22T23:55:01,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:01,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:01,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701988) roll requested 2024-11-22T23:55:01,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319701994 2024-11-22T23:55:02,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701994 2024-11-22T23:55:02,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319701994) roll requested 2024-11-22T23:55:02,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702001 2024-11-22T23:55:02,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319701994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702001 2024-11-22T23:55:02,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702001) roll requested 2024-11-22T23:55:02,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702008 2024-11-22T23:55:02,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702008 2024-11-22T23:55:02,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702008) roll requested 2024-11-22T23:55:02,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702016 2024-11-22T23:55:02,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702016 2024-11-22T23:55:02,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702016) roll requested 2024-11-22T23:55:02,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702024 2024-11-22T23:55:02,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702024 2024-11-22T23:55:02,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702024) roll requested 2024-11-22T23:55:02,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702031 2024-11-22T23:55:02,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702031 2024-11-22T23:55:02,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702031) roll requested 2024-11-22T23:55:02,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702038 2024-11-22T23:55:02,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702038 2024-11-22T23:55:02,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702038) roll requested 2024-11-22T23:55:02,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702045 2024-11-22T23:55:02,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702045 2024-11-22T23:55:02,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702045) roll requested 2024-11-22T23:55:02,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702053 2024-11-22T23:55:02,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702053 2024-11-22T23:55:02,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702053) roll requested 2024-11-22T23:55:02,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702060 2024-11-22T23:55:02,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702060 2024-11-22T23:55:02,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702060) roll requested 2024-11-22T23:55:02,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702067 2024-11-22T23:55:02,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702067 2024-11-22T23:55:02,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702067) roll requested 2024-11-22T23:55:02,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702074 2024-11-22T23:55:02,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702074 2024-11-22T23:55:02,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702074) roll requested 2024-11-22T23:55:02,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702081 2024-11-22T23:55:02,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702081 2024-11-22T23:55:02,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702081) roll requested 2024-11-22T23:55:02,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702088 2024-11-22T23:55:02,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702088 2024-11-22T23:55:02,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702088) roll requested 2024-11-22T23:55:02,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702094 2024-11-22T23:55:02,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702094 2024-11-22T23:55:02,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702094) roll requested 2024-11-22T23:55:02,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702102 2024-11-22T23:55:02,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702102 2024-11-22T23:55:02,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702102) roll requested 2024-11-22T23:55:02,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702109 2024-11-22T23:55:02,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702109 2024-11-22T23:55:02,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702109) roll requested 2024-11-22T23:55:02,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702115 2024-11-22T23:55:02,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702115 2024-11-22T23:55:02,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702115) roll requested 2024-11-22T23:55:02,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702122 2024-11-22T23:55:02,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702122 2024-11-22T23:55:02,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702122) roll requested 2024-11-22T23:55:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702129 2024-11-22T23:55:02,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702129 2024-11-22T23:55:02,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702129) roll requested 2024-11-22T23:55:02,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702136 2024-11-22T23:55:02,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702136 2024-11-22T23:55:02,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702136) roll requested 2024-11-22T23:55:02,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702143 2024-11-22T23:55:02,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702143 2024-11-22T23:55:02,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702143) roll requested 2024-11-22T23:55:02,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702151 2024-11-22T23:55:02,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702151 2024-11-22T23:55:02,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702151) roll requested 2024-11-22T23:55:02,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702158 2024-11-22T23:55:02,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702158 2024-11-22T23:55:02,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702158) roll requested 2024-11-22T23:55:02,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702165 2024-11-22T23:55:02,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702165 2024-11-22T23:55:02,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702165) roll requested 2024-11-22T23:55:02,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702172 2024-11-22T23:55:02,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702172 2024-11-22T23:55:02,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702172) roll requested 2024-11-22T23:55:02,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702179 2024-11-22T23:55:02,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702179 2024-11-22T23:55:02,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702179) roll requested 2024-11-22T23:55:02,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702186 2024-11-22T23:55:02,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702186 2024-11-22T23:55:02,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702186) roll requested 2024-11-22T23:55:02,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702193 2024-11-22T23:55:02,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702193 2024-11-22T23:55:02,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702193) roll requested 2024-11-22T23:55:02,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702200 2024-11-22T23:55:02,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702200 2024-11-22T23:55:02,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702200) roll requested 2024-11-22T23:55:02,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702207 2024-11-22T23:55:02,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702207 2024-11-22T23:55:02,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702207) roll requested 2024-11-22T23:55:02,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702213 2024-11-22T23:55:02,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702213 2024-11-22T23:55:02,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702213) roll requested 2024-11-22T23:55:02,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702221 2024-11-22T23:55:02,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702221 2024-11-22T23:55:02,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702221) roll requested 2024-11-22T23:55:02,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702230 2024-11-22T23:55:02,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702230 2024-11-22T23:55:02,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702230) roll requested 2024-11-22T23:55:02,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702237 2024-11-22T23:55:02,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702237 2024-11-22T23:55:02,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702237) roll requested 2024-11-22T23:55:02,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702244 2024-11-22T23:55:02,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702244 2024-11-22T23:55:02,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702244) roll requested 2024-11-22T23:55:02,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702252 2024-11-22T23:55:02,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702252 2024-11-22T23:55:02,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702252) roll requested 2024-11-22T23:55:02,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702267 2024-11-22T23:55:02,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702267 2024-11-22T23:55:02,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702267) roll requested 2024-11-22T23:55:02,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702275 2024-11-22T23:55:02,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702275 2024-11-22T23:55:02,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702275) roll requested 2024-11-22T23:55:02,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702282 2024-11-22T23:55:02,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702282 2024-11-22T23:55:02,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702282) roll requested 2024-11-22T23:55:02,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702290 2024-11-22T23:55:02,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702290 2024-11-22T23:55:02,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702290) roll requested 2024-11-22T23:55:02,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702297 2024-11-22T23:55:02,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702297 2024-11-22T23:55:02,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702297) roll requested 2024-11-22T23:55:02,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702303 2024-11-22T23:55:02,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702303 2024-11-22T23:55:02,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702303) roll requested 2024-11-22T23:55:02,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702310 2024-11-22T23:55:02,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702310 2024-11-22T23:55:02,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702310) roll requested 2024-11-22T23:55:02,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702317 2024-11-22T23:55:02,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702317 2024-11-22T23:55:02,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702317) roll requested 2024-11-22T23:55:02,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702324 2024-11-22T23:55:02,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702324 2024-11-22T23:55:02,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702324) roll requested 2024-11-22T23:55:02,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702331 2024-11-22T23:55:02,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702331 2024-11-22T23:55:02,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702331) roll requested 2024-11-22T23:55:02,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702338 2024-11-22T23:55:02,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702338 2024-11-22T23:55:02,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702338) roll requested 2024-11-22T23:55:02,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702345 2024-11-22T23:55:02,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702345 2024-11-22T23:55:02,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702345) roll requested 2024-11-22T23:55:02,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702362 2024-11-22T23:55:02,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702362 2024-11-22T23:55:02,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702362) roll requested 2024-11-22T23:55:02,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702370 2024-11-22T23:55:02,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702370 2024-11-22T23:55:02,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702370) roll requested 2024-11-22T23:55:02,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702377 2024-11-22T23:55:02,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702377 2024-11-22T23:55:02,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702377) roll requested 2024-11-22T23:55:02,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702384 2024-11-22T23:55:02,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702384 2024-11-22T23:55:02,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702384) roll requested 2024-11-22T23:55:02,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702395 2024-11-22T23:55:02,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702395 2024-11-22T23:55:02,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702395) roll requested 2024-11-22T23:55:02,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702403 2024-11-22T23:55:02,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702403 2024-11-22T23:55:02,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702403) roll requested 2024-11-22T23:55:02,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702410 2024-11-22T23:55:02,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702410 2024-11-22T23:55:02,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702410) roll requested 2024-11-22T23:55:02,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702418 2024-11-22T23:55:02,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702418 2024-11-22T23:55:02,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702418) roll requested 2024-11-22T23:55:02,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702425 2024-11-22T23:55:02,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702425 2024-11-22T23:55:02,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702425) roll requested 2024-11-22T23:55:02,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702432 2024-11-22T23:55:02,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702432 2024-11-22T23:55:02,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702432) roll requested 2024-11-22T23:55:02,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702440 2024-11-22T23:55:02,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702440 2024-11-22T23:55:02,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702440) roll requested 2024-11-22T23:55:02,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702446 2024-11-22T23:55:02,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702446 2024-11-22T23:55:02,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702446) roll requested 2024-11-22T23:55:02,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702453 2024-11-22T23:55:02,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702453 2024-11-22T23:55:02,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702453) roll requested 2024-11-22T23:55:02,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702460 2024-11-22T23:55:02,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702460 2024-11-22T23:55:02,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702460) roll requested 2024-11-22T23:55:02,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702467 2024-11-22T23:55:02,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702467 2024-11-22T23:55:02,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702467) roll requested 2024-11-22T23:55:02,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702474 2024-11-22T23:55:02,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702474 2024-11-22T23:55:02,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702474) roll requested 2024-11-22T23:55:02,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702481 2024-11-22T23:55:02,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702481 2024-11-22T23:55:02,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702481) roll requested 2024-11-22T23:55:02,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702488 2024-11-22T23:55:02,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702488 2024-11-22T23:55:02,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702488) roll requested 2024-11-22T23:55:02,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702497 2024-11-22T23:55:02,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702497 2024-11-22T23:55:02,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702497) roll requested 2024-11-22T23:55:02,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702504 2024-11-22T23:55:02,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702504 2024-11-22T23:55:02,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702504) roll requested 2024-11-22T23:55:02,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702520 2024-11-22T23:55:02,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702520 2024-11-22T23:55:02,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702520) roll requested 2024-11-22T23:55:02,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702528 2024-11-22T23:55:02,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702528 2024-11-22T23:55:02,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702528) roll requested 2024-11-22T23:55:02,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702535 2024-11-22T23:55:02,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702535 2024-11-22T23:55:02,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702535) roll requested 2024-11-22T23:55:02,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702542 2024-11-22T23:55:02,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702542 2024-11-22T23:55:02,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702542) roll requested 2024-11-22T23:55:02,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702549 2024-11-22T23:55:02,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702549 2024-11-22T23:55:02,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702549) roll requested 2024-11-22T23:55:02,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702556 2024-11-22T23:55:02,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702556 2024-11-22T23:55:02,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702556) roll requested 2024-11-22T23:55:02,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702563 2024-11-22T23:55:02,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702563 2024-11-22T23:55:02,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702563) roll requested 2024-11-22T23:55:02,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702571 2024-11-22T23:55:02,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702571 2024-11-22T23:55:02,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702571) roll requested 2024-11-22T23:55:02,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702578 2024-11-22T23:55:02,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702578 2024-11-22T23:55:02,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702578) roll requested 2024-11-22T23:55:02,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702585 2024-11-22T23:55:02,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702585 2024-11-22T23:55:02,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702585) roll requested 2024-11-22T23:55:02,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702592 2024-11-22T23:55:02,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702592 2024-11-22T23:55:02,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702592) roll requested 2024-11-22T23:55:02,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702602 2024-11-22T23:55:02,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702602 2024-11-22T23:55:02,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702602) roll requested 2024-11-22T23:55:02,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702614 2024-11-22T23:55:02,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702614 2024-11-22T23:55:02,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702614) roll requested 2024-11-22T23:55:02,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702622 2024-11-22T23:55:02,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702622 2024-11-22T23:55:02,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702622) roll requested 2024-11-22T23:55:02,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702629 2024-11-22T23:55:02,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702629 2024-11-22T23:55:02,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702629) roll requested 2024-11-22T23:55:02,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702637 2024-11-22T23:55:02,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702637 2024-11-22T23:55:02,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702637) roll requested 2024-11-22T23:55:02,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702644 2024-11-22T23:55:02,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702644 2024-11-22T23:55:02,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702644) roll requested 2024-11-22T23:55:02,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702651 2024-11-22T23:55:02,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702651 2024-11-22T23:55:02,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702651) roll requested 2024-11-22T23:55:02,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702659 2024-11-22T23:55:02,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702659 2024-11-22T23:55:02,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702659) roll requested 2024-11-22T23:55:02,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702666 2024-11-22T23:55:02,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702666 2024-11-22T23:55:02,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702666) roll requested 2024-11-22T23:55:02,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702676 2024-11-22T23:55:02,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702676 2024-11-22T23:55:02,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702676) roll requested 2024-11-22T23:55:02,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702683 2024-11-22T23:55:02,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702683 2024-11-22T23:55:02,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702683) roll requested 2024-11-22T23:55:02,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702690 2024-11-22T23:55:02,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702690 2024-11-22T23:55:02,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702690) roll requested 2024-11-22T23:55:02,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702697 2024-11-22T23:55:02,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702697 2024-11-22T23:55:02,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702697) roll requested 2024-11-22T23:55:02,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702705 2024-11-22T23:55:02,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702705 2024-11-22T23:55:02,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702705) roll requested 2024-11-22T23:55:02,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702712 2024-11-22T23:55:02,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702712 2024-11-22T23:55:02,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702712) roll requested 2024-11-22T23:55:02,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702719 2024-11-22T23:55:02,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702719 2024-11-22T23:55:02,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702719) roll requested 2024-11-22T23:55:02,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702728 2024-11-22T23:55:02,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702728 2024-11-22T23:55:02,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702728) roll requested 2024-11-22T23:55:02,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702736 2024-11-22T23:55:02,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702736 2024-11-22T23:55:02,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702736) roll requested 2024-11-22T23:55:02,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702743 2024-11-22T23:55:02,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702743 2024-11-22T23:55:02,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702743) roll requested 2024-11-22T23:55:02,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702750 2024-11-22T23:55:02,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702750 2024-11-22T23:55:02,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702750) roll requested 2024-11-22T23:55:02,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702758 2024-11-22T23:55:02,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702758 2024-11-22T23:55:02,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702758) roll requested 2024-11-22T23:55:02,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702766 2024-11-22T23:55:02,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702766 2024-11-22T23:55:02,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702766) roll requested 2024-11-22T23:55:02,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702773 2024-11-22T23:55:02,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702773 2024-11-22T23:55:02,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702773) roll requested 2024-11-22T23:55:02,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702781 2024-11-22T23:55:02,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702781 2024-11-22T23:55:02,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702781) roll requested 2024-11-22T23:55:02,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702789 2024-11-22T23:55:02,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702789 2024-11-22T23:55:02,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702789) roll requested 2024-11-22T23:55:02,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702796 2024-11-22T23:55:02,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702796 2024-11-22T23:55:02,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702796) roll requested 2024-11-22T23:55:02,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702804 2024-11-22T23:55:02,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702804 2024-11-22T23:55:02,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702804) roll requested 2024-11-22T23:55:02,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702816 2024-11-22T23:55:02,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702816 2024-11-22T23:55:02,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702816) roll requested 2024-11-22T23:55:02,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702825 2024-11-22T23:55:02,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702825 2024-11-22T23:55:02,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702825) roll requested 2024-11-22T23:55:02,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702832 2024-11-22T23:55:02,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702832 2024-11-22T23:55:02,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702832) roll requested 2024-11-22T23:55:02,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702840 2024-11-22T23:55:02,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702840 2024-11-22T23:55:02,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702840) roll requested 2024-11-22T23:55:02,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702848 2024-11-22T23:55:02,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702848 2024-11-22T23:55:02,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702848) roll requested 2024-11-22T23:55:02,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702855 2024-11-22T23:55:02,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702855 2024-11-22T23:55:02,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702855) roll requested 2024-11-22T23:55:02,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702863 2024-11-22T23:55:02,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702863 2024-11-22T23:55:02,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702863) roll requested 2024-11-22T23:55:02,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702869 2024-11-22T23:55:02,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702869 2024-11-22T23:55:02,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702869) roll requested 2024-11-22T23:55:02,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702876 2024-11-22T23:55:02,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702876 2024-11-22T23:55:02,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702876) roll requested 2024-11-22T23:55:02,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702883 2024-11-22T23:55:02,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702883 2024-11-22T23:55:02,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702883) roll requested 2024-11-22T23:55:02,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702890 2024-11-22T23:55:02,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702890 2024-11-22T23:55:02,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702890) roll requested 2024-11-22T23:55:02,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702896 2024-11-22T23:55:02,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702896 2024-11-22T23:55:02,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702896) roll requested 2024-11-22T23:55:02,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702905 2024-11-22T23:55:02,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702905 2024-11-22T23:55:02,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702905) roll requested 2024-11-22T23:55:02,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702912 2024-11-22T23:55:02,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702912 2024-11-22T23:55:02,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702912) roll requested 2024-11-22T23:55:02,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702919 2024-11-22T23:55:02,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702919 2024-11-22T23:55:02,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702919) roll requested 2024-11-22T23:55:02,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702926 2024-11-22T23:55:02,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702926 2024-11-22T23:55:02,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702926) roll requested 2024-11-22T23:55:02,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702933 2024-11-22T23:55:02,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702933 2024-11-22T23:55:02,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702933) roll requested 2024-11-22T23:55:02,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702940 2024-11-22T23:55:02,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702940 2024-11-22T23:55:02,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702940) roll requested 2024-11-22T23:55:02,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702947 2024-11-22T23:55:02,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702947 2024-11-22T23:55:02,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702947) roll requested 2024-11-22T23:55:02,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702954 2024-11-22T23:55:02,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702954 2024-11-22T23:55:02,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702954) roll requested 2024-11-22T23:55:02,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702961 2024-11-22T23:55:02,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702961 2024-11-22T23:55:02,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702961) roll requested 2024-11-22T23:55:02,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702968 2024-11-22T23:55:02,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702968 2024-11-22T23:55:02,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702968) roll requested 2024-11-22T23:55:02,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702975 2024-11-22T23:55:02,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702975 2024-11-22T23:55:02,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702975) roll requested 2024-11-22T23:55:02,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702982 2024-11-22T23:55:02,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702982 2024-11-22T23:55:02,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702982) roll requested 2024-11-22T23:55:02,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702989 2024-11-22T23:55:02,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:02,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702989 2024-11-22T23:55:02,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:02,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:02,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702989) roll requested 2024-11-22T23:55:02,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319702996 2024-11-22T23:55:03,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702996 2024-11-22T23:55:03,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319702996) roll requested 2024-11-22T23:55:03,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703003 2024-11-22T23:55:03,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319702996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703003 2024-11-22T23:55:03,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703003) roll requested 2024-11-22T23:55:03,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703012 2024-11-22T23:55:03,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703012 2024-11-22T23:55:03,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703012) roll requested 2024-11-22T23:55:03,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703019 2024-11-22T23:55:03,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703019 2024-11-22T23:55:03,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703019) roll requested 2024-11-22T23:55:03,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703026 2024-11-22T23:55:03,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703026 2024-11-22T23:55:03,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703026) roll requested 2024-11-22T23:55:03,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703033 2024-11-22T23:55:03,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703033 2024-11-22T23:55:03,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703033) roll requested 2024-11-22T23:55:03,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703041 2024-11-22T23:55:03,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703041 2024-11-22T23:55:03,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703041) roll requested 2024-11-22T23:55:03,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703048 2024-11-22T23:55:03,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703048 2024-11-22T23:55:03,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703048) roll requested 2024-11-22T23:55:03,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703055 2024-11-22T23:55:03,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703055 2024-11-22T23:55:03,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703055) roll requested 2024-11-22T23:55:03,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703062 2024-11-22T23:55:03,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703062 2024-11-22T23:55:03,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703062) roll requested 2024-11-22T23:55:03,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703070 2024-11-22T23:55:03,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703070 2024-11-22T23:55:03,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703070) roll requested 2024-11-22T23:55:03,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703077 2024-11-22T23:55:03,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703077 2024-11-22T23:55:03,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703077) roll requested 2024-11-22T23:55:03,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703084 2024-11-22T23:55:03,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703084 2024-11-22T23:55:03,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703084) roll requested 2024-11-22T23:55:03,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703099 2024-11-22T23:55:03,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703099 2024-11-22T23:55:03,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703099) roll requested 2024-11-22T23:55:03,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703106 2024-11-22T23:55:03,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703106 2024-11-22T23:55:03,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703106) roll requested 2024-11-22T23:55:03,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703114 2024-11-22T23:55:03,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703114 2024-11-22T23:55:03,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703114) roll requested 2024-11-22T23:55:03,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703123 2024-11-22T23:55:03,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703123 2024-11-22T23:55:03,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703123) roll requested 2024-11-22T23:55:03,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703130 2024-11-22T23:55:03,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703130 2024-11-22T23:55:03,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703130) roll requested 2024-11-22T23:55:03,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703137 2024-11-22T23:55:03,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703137 2024-11-22T23:55:03,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703137) roll requested 2024-11-22T23:55:03,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703145 2024-11-22T23:55:03,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703145 2024-11-22T23:55:03,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703145) roll requested 2024-11-22T23:55:03,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703152 2024-11-22T23:55:03,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703152 2024-11-22T23:55:03,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703152) roll requested 2024-11-22T23:55:03,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703159 2024-11-22T23:55:03,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703159 2024-11-22T23:55:03,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703159) roll requested 2024-11-22T23:55:03,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703165 2024-11-22T23:55:03,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703165 2024-11-22T23:55:03,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703165) roll requested 2024-11-22T23:55:03,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703173 2024-11-22T23:55:03,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703173 2024-11-22T23:55:03,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703173) roll requested 2024-11-22T23:55:03,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703180 2024-11-22T23:55:03,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703180 2024-11-22T23:55:03,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703180) roll requested 2024-11-22T23:55:03,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703187 2024-11-22T23:55:03,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703187 2024-11-22T23:55:03,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703187) roll requested 2024-11-22T23:55:03,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703194 2024-11-22T23:55:03,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703194 2024-11-22T23:55:03,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703194) roll requested 2024-11-22T23:55:03,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703201 2024-11-22T23:55:03,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703201 2024-11-22T23:55:03,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703201) roll requested 2024-11-22T23:55:03,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703208 2024-11-22T23:55:03,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703208 2024-11-22T23:55:03,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703208) roll requested 2024-11-22T23:55:03,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703216 2024-11-22T23:55:03,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703216 2024-11-22T23:55:03,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703216) roll requested 2024-11-22T23:55:03,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703224 2024-11-22T23:55:03,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703224 2024-11-22T23:55:03,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703224) roll requested 2024-11-22T23:55:03,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703231 2024-11-22T23:55:03,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703231 2024-11-22T23:55:03,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703231) roll requested 2024-11-22T23:55:03,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703237 2024-11-22T23:55:03,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703237 2024-11-22T23:55:03,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703237) roll requested 2024-11-22T23:55:03,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703248 2024-11-22T23:55:03,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703248 2024-11-22T23:55:03,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703248) roll requested 2024-11-22T23:55:03,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703257 2024-11-22T23:55:03,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703257 2024-11-22T23:55:03,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703257) roll requested 2024-11-22T23:55:03,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703264 2024-11-22T23:55:03,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703264 2024-11-22T23:55:03,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703264) roll requested 2024-11-22T23:55:03,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703271 2024-11-22T23:55:03,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703271 2024-11-22T23:55:03,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703271) roll requested 2024-11-22T23:55:03,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703278 2024-11-22T23:55:03,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703278 2024-11-22T23:55:03,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703278) roll requested 2024-11-22T23:55:03,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703285 2024-11-22T23:55:03,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703278 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703285 2024-11-22T23:55:03,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703285) roll requested 2024-11-22T23:55:03,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703292 2024-11-22T23:55:03,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703292 2024-11-22T23:55:03,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703292) roll requested 2024-11-22T23:55:03,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703299 2024-11-22T23:55:03,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703299 2024-11-22T23:55:03,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703299) roll requested 2024-11-22T23:55:03,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703306 2024-11-22T23:55:03,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703306 2024-11-22T23:55:03,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703306) roll requested 2024-11-22T23:55:03,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703313 2024-11-22T23:55:03,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703313 2024-11-22T23:55:03,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703313) roll requested 2024-11-22T23:55:03,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703321 2024-11-22T23:55:03,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703321 2024-11-22T23:55:03,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703321) roll requested 2024-11-22T23:55:03,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703328 2024-11-22T23:55:03,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703328 2024-11-22T23:55:03,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703328) roll requested 2024-11-22T23:55:03,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703335 2024-11-22T23:55:03,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703335 2024-11-22T23:55:03,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703335) roll requested 2024-11-22T23:55:03,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703343 2024-11-22T23:55:03,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703343 2024-11-22T23:55:03,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703343) roll requested 2024-11-22T23:55:03,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703350 2024-11-22T23:55:03,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703350 2024-11-22T23:55:03,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703350) roll requested 2024-11-22T23:55:03,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703357 2024-11-22T23:55:03,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703357 2024-11-22T23:55:03,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703357) roll requested 2024-11-22T23:55:03,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703364 2024-11-22T23:55:03,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703364 2024-11-22T23:55:03,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703364) roll requested 2024-11-22T23:55:03,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703371 2024-11-22T23:55:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703371 2024-11-22T23:55:03,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703371) roll requested 2024-11-22T23:55:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703378 2024-11-22T23:55:03,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703378 2024-11-22T23:55:03,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703378) roll requested 2024-11-22T23:55:03,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703385 2024-11-22T23:55:03,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703385 2024-11-22T23:55:03,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703385) roll requested 2024-11-22T23:55:03,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703392 2024-11-22T23:55:03,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703392 2024-11-22T23:55:03,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703392) roll requested 2024-11-22T23:55:03,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703399 2024-11-22T23:55:03,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703399 2024-11-22T23:55:03,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703399) roll requested 2024-11-22T23:55:03,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703406 2024-11-22T23:55:03,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703406 2024-11-22T23:55:03,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703406) roll requested 2024-11-22T23:55:03,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703413 2024-11-22T23:55:03,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703413 2024-11-22T23:55:03,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703413) roll requested 2024-11-22T23:55:03,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703420 2024-11-22T23:55:03,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703420 2024-11-22T23:55:03,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703420) roll requested 2024-11-22T23:55:03,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703427 2024-11-22T23:55:03,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703427 2024-11-22T23:55:03,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703427) roll requested 2024-11-22T23:55:03,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703435 2024-11-22T23:55:03,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703435 2024-11-22T23:55:03,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703435) roll requested 2024-11-22T23:55:03,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703442 2024-11-22T23:55:03,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703442 2024-11-22T23:55:03,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703442) roll requested 2024-11-22T23:55:03,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703449 2024-11-22T23:55:03,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703449 2024-11-22T23:55:03,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703449) roll requested 2024-11-22T23:55:03,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703457 2024-11-22T23:55:03,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703457 2024-11-22T23:55:03,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703457) roll requested 2024-11-22T23:55:03,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703464 2024-11-22T23:55:03,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703464 2024-11-22T23:55:03,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703464) roll requested 2024-11-22T23:55:03,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703471 2024-11-22T23:55:03,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703471 2024-11-22T23:55:03,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703471) roll requested 2024-11-22T23:55:03,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703477 2024-11-22T23:55:03,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703477 2024-11-22T23:55:03,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703477) roll requested 2024-11-22T23:55:03,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703484 2024-11-22T23:55:03,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703484 2024-11-22T23:55:03,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703484) roll requested 2024-11-22T23:55:03,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703491 2024-11-22T23:55:03,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703491 2024-11-22T23:55:03,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703491) roll requested 2024-11-22T23:55:03,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703498 2024-11-22T23:55:03,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703498 2024-11-22T23:55:03,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703498) roll requested 2024-11-22T23:55:03,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703506 2024-11-22T23:55:03,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703506 2024-11-22T23:55:03,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703506) roll requested 2024-11-22T23:55:03,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703513 2024-11-22T23:55:03,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703513 2024-11-22T23:55:03,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703513) roll requested 2024-11-22T23:55:03,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703521 2024-11-22T23:55:03,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703521 2024-11-22T23:55:03,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703521) roll requested 2024-11-22T23:55:03,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703528 2024-11-22T23:55:03,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703528 2024-11-22T23:55:03,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703528) roll requested 2024-11-22T23:55:03,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703535 2024-11-22T23:55:03,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703535 2024-11-22T23:55:03,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703535) roll requested 2024-11-22T23:55:03,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703542 2024-11-22T23:55:03,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703542 2024-11-22T23:55:03,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703542) roll requested 2024-11-22T23:55:03,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703549 2024-11-22T23:55:03,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703549 2024-11-22T23:55:03,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703549) roll requested 2024-11-22T23:55:03,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703557 2024-11-22T23:55:03,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703557 2024-11-22T23:55:03,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703557) roll requested 2024-11-22T23:55:03,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703564 2024-11-22T23:55:03,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703564 2024-11-22T23:55:03,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703564) roll requested 2024-11-22T23:55:03,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703571 2024-11-22T23:55:03,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703571 2024-11-22T23:55:03,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703571) roll requested 2024-11-22T23:55:03,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703578 2024-11-22T23:55:03,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703578 2024-11-22T23:55:03,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703578) roll requested 2024-11-22T23:55:03,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703585 2024-11-22T23:55:03,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703585 2024-11-22T23:55:03,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703585) roll requested 2024-11-22T23:55:03,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703592 2024-11-22T23:55:03,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703592 2024-11-22T23:55:03,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703592) roll requested 2024-11-22T23:55:03,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703599 2024-11-22T23:55:03,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703599 2024-11-22T23:55:03,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703599) roll requested 2024-11-22T23:55:03,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703606 2024-11-22T23:55:03,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703606 2024-11-22T23:55:03,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703606) roll requested 2024-11-22T23:55:03,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703617 2024-11-22T23:55:03,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703617 2024-11-22T23:55:03,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703617) roll requested 2024-11-22T23:55:03,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703633 2024-11-22T23:55:03,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703633 2024-11-22T23:55:03,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703633) roll requested 2024-11-22T23:55:03,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703646 2024-11-22T23:55:03,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703646 2024-11-22T23:55:03,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703646) roll requested 2024-11-22T23:55:03,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703653 2024-11-22T23:55:03,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703653 2024-11-22T23:55:03,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703653) roll requested 2024-11-22T23:55:03,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703661 2024-11-22T23:55:03,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703661 2024-11-22T23:55:03,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703661) roll requested 2024-11-22T23:55:03,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703667 2024-11-22T23:55:03,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703667 2024-11-22T23:55:03,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703667) roll requested 2024-11-22T23:55:03,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703675 2024-11-22T23:55:03,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703675 2024-11-22T23:55:03,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703675) roll requested 2024-11-22T23:55:03,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703682 2024-11-22T23:55:03,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703682 2024-11-22T23:55:03,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703682) roll requested 2024-11-22T23:55:03,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703688 2024-11-22T23:55:03,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703688 2024-11-22T23:55:03,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703688) roll requested 2024-11-22T23:55:03,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703695 2024-11-22T23:55:03,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703695 2024-11-22T23:55:03,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703695) roll requested 2024-11-22T23:55:03,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703702 2024-11-22T23:55:03,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703702 2024-11-22T23:55:03,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703702) roll requested 2024-11-22T23:55:03,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703709 2024-11-22T23:55:03,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703709 2024-11-22T23:55:03,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703709) roll requested 2024-11-22T23:55:03,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703715 2024-11-22T23:55:03,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703715 2024-11-22T23:55:03,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703715) roll requested 2024-11-22T23:55:03,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703723 2024-11-22T23:55:03,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703723 2024-11-22T23:55:03,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703723) roll requested 2024-11-22T23:55:03,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703729 2024-11-22T23:55:03,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703729 2024-11-22T23:55:03,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703729) roll requested 2024-11-22T23:55:03,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703736 2024-11-22T23:55:03,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703736 2024-11-22T23:55:03,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703736) roll requested 2024-11-22T23:55:03,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703743 2024-11-22T23:55:03,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703743 2024-11-22T23:55:03,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703743) roll requested 2024-11-22T23:55:03,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703751 2024-11-22T23:55:03,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703751 2024-11-22T23:55:03,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703751) roll requested 2024-11-22T23:55:03,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703758 2024-11-22T23:55:03,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703758 2024-11-22T23:55:03,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703758) roll requested 2024-11-22T23:55:03,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703766 2024-11-22T23:55:03,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703766 2024-11-22T23:55:03,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703766) roll requested 2024-11-22T23:55:03,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703773 2024-11-22T23:55:03,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703773 2024-11-22T23:55:03,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703773) roll requested 2024-11-22T23:55:03,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703780 2024-11-22T23:55:03,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703780 2024-11-22T23:55:03,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703780) roll requested 2024-11-22T23:55:03,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703786 2024-11-22T23:55:03,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703786 2024-11-22T23:55:03,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703786) roll requested 2024-11-22T23:55:03,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703793 2024-11-22T23:55:03,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703793 2024-11-22T23:55:03,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703793) roll requested 2024-11-22T23:55:03,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703800 2024-11-22T23:55:03,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703800 2024-11-22T23:55:03,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703800) roll requested 2024-11-22T23:55:03,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703807 2024-11-22T23:55:03,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703807 2024-11-22T23:55:03,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703807) roll requested 2024-11-22T23:55:03,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703814 2024-11-22T23:55:03,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703814 2024-11-22T23:55:03,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703814) roll requested 2024-11-22T23:55:03,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703822 2024-11-22T23:55:03,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703822 2024-11-22T23:55:03,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703822) roll requested 2024-11-22T23:55:03,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703829 2024-11-22T23:55:03,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703829 2024-11-22T23:55:03,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703829) roll requested 2024-11-22T23:55:03,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703836 2024-11-22T23:55:03,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703836 2024-11-22T23:55:03,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703836) roll requested 2024-11-22T23:55:03,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703843 2024-11-22T23:55:03,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703843 2024-11-22T23:55:03,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703843) roll requested 2024-11-22T23:55:03,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703851 2024-11-22T23:55:03,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703851 2024-11-22T23:55:03,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703851) roll requested 2024-11-22T23:55:03,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703858 2024-11-22T23:55:03,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703858 2024-11-22T23:55:03,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703858) roll requested 2024-11-22T23:55:03,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703865 2024-11-22T23:55:03,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703865 2024-11-22T23:55:03,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703865) roll requested 2024-11-22T23:55:03,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703872 2024-11-22T23:55:03,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703872 2024-11-22T23:55:03,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703872) roll requested 2024-11-22T23:55:03,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703880 2024-11-22T23:55:03,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703880 2024-11-22T23:55:03,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703880) roll requested 2024-11-22T23:55:03,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703890 2024-11-22T23:55:03,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703890 2024-11-22T23:55:03,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703890) roll requested 2024-11-22T23:55:03,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703903 2024-11-22T23:55:03,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703903 2024-11-22T23:55:03,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703903) roll requested 2024-11-22T23:55:03,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703928 2024-11-22T23:55:03,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703928 2024-11-22T23:55:03,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703928) roll requested 2024-11-22T23:55:03,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703943 2024-11-22T23:55:03,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703943 2024-11-22T23:55:03,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703943) roll requested 2024-11-22T23:55:03,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703953 2024-11-22T23:55:03,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703953 2024-11-22T23:55:03,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703953) roll requested 2024-11-22T23:55:03,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703966 2024-11-22T23:55:03,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703966 2024-11-22T23:55:03,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703966) roll requested 2024-11-22T23:55:03,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703980 2024-11-22T23:55:03,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703980 2024-11-22T23:55:03,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703980) roll requested 2024-11-22T23:55:03,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703988 2024-11-22T23:55:03,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:03,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703988 2024-11-22T23:55:03,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:03,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:03,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703988) roll requested 2024-11-22T23:55:03,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319703998 2024-11-22T23:55:04,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703998 2024-11-22T23:55:04,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319703998) roll requested 2024-11-22T23:55:04,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704006 2024-11-22T23:55:04,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319703998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704006 2024-11-22T23:55:04,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704006) roll requested 2024-11-22T23:55:04,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704017 2024-11-22T23:55:04,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704017 2024-11-22T23:55:04,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704017) roll requested 2024-11-22T23:55:04,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704025 2024-11-22T23:55:04,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704025 2024-11-22T23:55:04,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704025) roll requested 2024-11-22T23:55:04,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704042 2024-11-22T23:55:04,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704042 2024-11-22T23:55:04,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704042) roll requested 2024-11-22T23:55:04,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704057 2024-11-22T23:55:04,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704057 2024-11-22T23:55:04,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704057) roll requested 2024-11-22T23:55:04,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704065 2024-11-22T23:55:04,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704065 2024-11-22T23:55:04,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704065) roll requested 2024-11-22T23:55:04,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704077 2024-11-22T23:55:04,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704077 2024-11-22T23:55:04,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704077) roll requested 2024-11-22T23:55:04,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704089 2024-11-22T23:55:04,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704089 2024-11-22T23:55:04,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704089) roll requested 2024-11-22T23:55:04,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704109 2024-11-22T23:55:04,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704109 2024-11-22T23:55:04,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704109) roll requested 2024-11-22T23:55:04,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704140 2024-11-22T23:55:04,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704140 2024-11-22T23:55:04,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704140) roll requested 2024-11-22T23:55:04,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704150 2024-11-22T23:55:04,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704150 2024-11-22T23:55:04,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704150) roll requested 2024-11-22T23:55:04,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704169 2024-11-22T23:55:04,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704169 2024-11-22T23:55:04,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704169) roll requested 2024-11-22T23:55:04,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704209 2024-11-22T23:55:04,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704209 2024-11-22T23:55:04,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704209) roll requested 2024-11-22T23:55:04,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704229 2024-11-22T23:55:04,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704229 2024-11-22T23:55:04,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704229) roll requested 2024-11-22T23:55:04,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704253 2024-11-22T23:55:04,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704253 2024-11-22T23:55:04,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704253) roll requested 2024-11-22T23:55:04,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704277 2024-11-22T23:55:04,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704277 2024-11-22T23:55:04,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704277) roll requested 2024-11-22T23:55:04,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704293 2024-11-22T23:55:04,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704293 2024-11-22T23:55:04,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704293) roll requested 2024-11-22T23:55:04,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704303 2024-11-22T23:55:04,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704303 2024-11-22T23:55:04,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704303) roll requested 2024-11-22T23:55:04,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704312 2024-11-22T23:55:04,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704312 2024-11-22T23:55:04,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704312) roll requested 2024-11-22T23:55:04,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704321 2024-11-22T23:55:04,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704321 2024-11-22T23:55:04,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704321) roll requested 2024-11-22T23:55:04,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704332 2024-11-22T23:55:04,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704332 2024-11-22T23:55:04,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704332) roll requested 2024-11-22T23:55:04,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704340 2024-11-22T23:55:04,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704340 2024-11-22T23:55:04,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704340) roll requested 2024-11-22T23:55:04,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704349 2024-11-22T23:55:04,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704349 2024-11-22T23:55:04,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704349) roll requested 2024-11-22T23:55:04,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704356 2024-11-22T23:55:04,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704356 2024-11-22T23:55:04,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704356) roll requested 2024-11-22T23:55:04,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704368 2024-11-22T23:55:04,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704368 2024-11-22T23:55:04,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704368) roll requested 2024-11-22T23:55:04,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704376 2024-11-22T23:55:04,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704376 2024-11-22T23:55:04,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704376) roll requested 2024-11-22T23:55:04,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704388 2024-11-22T23:55:04,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704388 2024-11-22T23:55:04,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704388) roll requested 2024-11-22T23:55:04,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704400 2024-11-22T23:55:04,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704400 2024-11-22T23:55:04,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704400) roll requested 2024-11-22T23:55:04,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704416 2024-11-22T23:55:04,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704416 2024-11-22T23:55:04,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704416) roll requested 2024-11-22T23:55:04,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704424 2024-11-22T23:55:04,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704424 2024-11-22T23:55:04,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704424) roll requested 2024-11-22T23:55:04,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704437 2024-11-22T23:55:04,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704437 2024-11-22T23:55:04,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704437) roll requested 2024-11-22T23:55:04,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704444 2024-11-22T23:55:04,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704444 2024-11-22T23:55:04,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704444) roll requested 2024-11-22T23:55:04,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704452 2024-11-22T23:55:04,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704452 2024-11-22T23:55:04,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704452) roll requested 2024-11-22T23:55:04,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704459 2024-11-22T23:55:04,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704459 2024-11-22T23:55:04,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704459) roll requested 2024-11-22T23:55:04,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704474 2024-11-22T23:55:04,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704474 2024-11-22T23:55:04,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704474) roll requested 2024-11-22T23:55:04,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704482 2024-11-22T23:55:04,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704482 2024-11-22T23:55:04,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704482) roll requested 2024-11-22T23:55:04,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704488 2024-11-22T23:55:04,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704488 2024-11-22T23:55:04,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704488) roll requested 2024-11-22T23:55:04,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704509 2024-11-22T23:55:04,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704509 2024-11-22T23:55:04,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704509) roll requested 2024-11-22T23:55:04,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704519 2024-11-22T23:55:04,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704519 2024-11-22T23:55:04,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704519) roll requested 2024-11-22T23:55:04,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704537 2024-11-22T23:55:04,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704537 2024-11-22T23:55:04,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704537) roll requested 2024-11-22T23:55:04,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704553 2024-11-22T23:55:04,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704553 2024-11-22T23:55:04,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704553) roll requested 2024-11-22T23:55:04,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704566 2024-11-22T23:55:04,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704566 2024-11-22T23:55:04,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704566) roll requested 2024-11-22T23:55:04,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704581 2024-11-22T23:55:04,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704581 2024-11-22T23:55:04,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704581) roll requested 2024-11-22T23:55:04,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704593 2024-11-22T23:55:04,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704593 2024-11-22T23:55:04,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704593) roll requested 2024-11-22T23:55:04,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704602 2024-11-22T23:55:04,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704602 2024-11-22T23:55:04,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704602) roll requested 2024-11-22T23:55:04,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704609 2024-11-22T23:55:04,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704609 2024-11-22T23:55:04,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704609) roll requested 2024-11-22T23:55:04,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704621 2024-11-22T23:55:04,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704621 2024-11-22T23:55:04,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704621) roll requested 2024-11-22T23:55:04,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704629 2024-11-22T23:55:04,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704629 2024-11-22T23:55:04,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704629) roll requested 2024-11-22T23:55:04,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704642 2024-11-22T23:55:04,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704642 2024-11-22T23:55:04,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704642) roll requested 2024-11-22T23:55:04,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704650 2024-11-22T23:55:04,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704650 2024-11-22T23:55:04,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704650) roll requested 2024-11-22T23:55:04,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704657 2024-11-22T23:55:04,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704657 2024-11-22T23:55:04,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704657) roll requested 2024-11-22T23:55:04,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704665 2024-11-22T23:55:04,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704665 2024-11-22T23:55:04,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704665) roll requested 2024-11-22T23:55:04,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704673 2024-11-22T23:55:04,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704673 2024-11-22T23:55:04,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704673) roll requested 2024-11-22T23:55:04,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704681 2024-11-22T23:55:04,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704681 2024-11-22T23:55:04,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704681) roll requested 2024-11-22T23:55:04,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704688 2024-11-22T23:55:04,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704688 2024-11-22T23:55:04,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704688) roll requested 2024-11-22T23:55:04,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704695 2024-11-22T23:55:04,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704695 2024-11-22T23:55:04,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704695) roll requested 2024-11-22T23:55:04,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704702 2024-11-22T23:55:04,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704702 2024-11-22T23:55:04,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704702) roll requested 2024-11-22T23:55:04,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704709 2024-11-22T23:55:04,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704709 2024-11-22T23:55:04,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704709) roll requested 2024-11-22T23:55:04,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704716 2024-11-22T23:55:04,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704716 2024-11-22T23:55:04,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704716) roll requested 2024-11-22T23:55:04,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704742 2024-11-22T23:55:04,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704742 2024-11-22T23:55:04,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704742) roll requested 2024-11-22T23:55:04,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704750 2024-11-22T23:55:04,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704750 2024-11-22T23:55:04,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704750) roll requested 2024-11-22T23:55:04,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704761 2024-11-22T23:55:04,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704761 2024-11-22T23:55:04,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704761) roll requested 2024-11-22T23:55:04,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704782 2024-11-22T23:55:04,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704782 2024-11-22T23:55:04,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704782) roll requested 2024-11-22T23:55:04,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704790 2024-11-22T23:55:04,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704790 2024-11-22T23:55:04,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704790) roll requested 2024-11-22T23:55:04,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704797 2024-11-22T23:55:04,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704797 2024-11-22T23:55:04,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704797) roll requested 2024-11-22T23:55:04,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704804 2024-11-22T23:55:04,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704804 2024-11-22T23:55:04,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704804) roll requested 2024-11-22T23:55:04,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704811 2024-11-22T23:55:04,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704811 2024-11-22T23:55:04,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704811) roll requested 2024-11-22T23:55:04,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704818 2024-11-22T23:55:04,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704818 2024-11-22T23:55:04,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704818) roll requested 2024-11-22T23:55:04,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704825 2024-11-22T23:55:04,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704825 2024-11-22T23:55:04,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704825) roll requested 2024-11-22T23:55:04,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704838 2024-11-22T23:55:04,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704838 2024-11-22T23:55:04,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704838) roll requested 2024-11-22T23:55:04,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704846 2024-11-22T23:55:04,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704846 2024-11-22T23:55:04,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704846) roll requested 2024-11-22T23:55:04,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704856 2024-11-22T23:55:04,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704856 2024-11-22T23:55:04,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704856) roll requested 2024-11-22T23:55:04,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704870 2024-11-22T23:55:04,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704870 2024-11-22T23:55:04,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704870) roll requested 2024-11-22T23:55:04,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704878 2024-11-22T23:55:04,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704878 2024-11-22T23:55:04,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704878) roll requested 2024-11-22T23:55:04,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704885 2024-11-22T23:55:04,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704885 2024-11-22T23:55:04,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704885) roll requested 2024-11-22T23:55:04,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704892 2024-11-22T23:55:04,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704892 2024-11-22T23:55:04,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704892) roll requested 2024-11-22T23:55:04,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704899 2024-11-22T23:55:04,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704899 2024-11-22T23:55:04,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704899) roll requested 2024-11-22T23:55:04,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704906 2024-11-22T23:55:04,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704906 2024-11-22T23:55:04,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704906) roll requested 2024-11-22T23:55:04,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704913 2024-11-22T23:55:04,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704913 2024-11-22T23:55:04,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704913) roll requested 2024-11-22T23:55:04,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704921 2024-11-22T23:55:04,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704921 2024-11-22T23:55:04,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704921) roll requested 2024-11-22T23:55:04,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704928 2024-11-22T23:55:04,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704928 2024-11-22T23:55:04,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704928) roll requested 2024-11-22T23:55:04,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704936 2024-11-22T23:55:04,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704936 2024-11-22T23:55:04,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704936) roll requested 2024-11-22T23:55:04,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704943 2024-11-22T23:55:04,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704943 2024-11-22T23:55:04,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704936 is not closed yet, will try archiving it next time 2024-11-22T23:55:04,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704943) roll requested 2024-11-22T23:55:04,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704951 2024-11-22T23:55:04,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704951 2024-11-22T23:55:04,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704951) roll requested 2024-11-22T23:55:04,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704961 2024-11-22T23:55:04,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704961 2024-11-22T23:55:04,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704961) roll requested 2024-11-22T23:55:04,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704969 2024-11-22T23:55:04,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704969 2024-11-22T23:55:04,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704969) roll requested 2024-11-22T23:55:04,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704976 2024-11-22T23:55:04,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704976 2024-11-22T23:55:04,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704976) roll requested 2024-11-22T23:55:04,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704983 2024-11-22T23:55:04,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704983 2024-11-22T23:55:04,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:04,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704983) roll requested 2024-11-22T23:55:04,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319704991 2024-11-22T23:55:04,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:04,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704991 2024-11-22T23:55:05,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:04,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319704991) roll requested 2024-11-22T23:55:05,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705000 2024-11-22T23:55:05,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319704991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705000 2024-11-22T23:55:05,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705000) roll requested 2024-11-22T23:55:05,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705009 2024-11-22T23:55:05,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705009 2024-11-22T23:55:05,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705009) roll requested 2024-11-22T23:55:05,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705021 2024-11-22T23:55:05,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705021 2024-11-22T23:55:05,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705021) roll requested 2024-11-22T23:55:05,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705028 2024-11-22T23:55:05,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705028 2024-11-22T23:55:05,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705028) roll requested 2024-11-22T23:55:05,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705042 2024-11-22T23:55:05,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705042 2024-11-22T23:55:05,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705042) roll requested 2024-11-22T23:55:05,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705049 2024-11-22T23:55:05,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705049 2024-11-22T23:55:05,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705049) roll requested 2024-11-22T23:55:05,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705063 2024-11-22T23:55:05,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705063 2024-11-22T23:55:05,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705063) roll requested 2024-11-22T23:55:05,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705072 2024-11-22T23:55:05,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705072 2024-11-22T23:55:05,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705072) roll requested 2024-11-22T23:55:05,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705080 2024-11-22T23:55:05,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705080 2024-11-22T23:55:05,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705080) roll requested 2024-11-22T23:55:05,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705087 2024-11-22T23:55:05,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705087 2024-11-22T23:55:05,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705087) roll requested 2024-11-22T23:55:05,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705095 2024-11-22T23:55:05,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705095 2024-11-22T23:55:05,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705095) roll requested 2024-11-22T23:55:05,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705102 2024-11-22T23:55:05,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705102 2024-11-22T23:55:05,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705102) roll requested 2024-11-22T23:55:05,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705109 2024-11-22T23:55:05,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705109 2024-11-22T23:55:05,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705109) roll requested 2024-11-22T23:55:05,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705116 2024-11-22T23:55:05,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705116 2024-11-22T23:55:05,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705116) roll requested 2024-11-22T23:55:05,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705123 2024-11-22T23:55:05,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705123 2024-11-22T23:55:05,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705123) roll requested 2024-11-22T23:55:05,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705130 2024-11-22T23:55:05,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705130 2024-11-22T23:55:05,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705130) roll requested 2024-11-22T23:55:05,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705138 2024-11-22T23:55:05,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705138 2024-11-22T23:55:05,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705138) roll requested 2024-11-22T23:55:05,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705153 2024-11-22T23:55:05,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705153 2024-11-22T23:55:05,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705153) roll requested 2024-11-22T23:55:05,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705161 2024-11-22T23:55:05,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705161 2024-11-22T23:55:05,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705161) roll requested 2024-11-22T23:55:05,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705171 2024-11-22T23:55:05,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705171 2024-11-22T23:55:05,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705171) roll requested 2024-11-22T23:55:05,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705181 2024-11-22T23:55:05,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705181 2024-11-22T23:55:05,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705181) roll requested 2024-11-22T23:55:05,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705193 2024-11-22T23:55:05,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705193 2024-11-22T23:55:05,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705193) roll requested 2024-11-22T23:55:05,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705201 2024-11-22T23:55:05,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705201 2024-11-22T23:55:05,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705201) roll requested 2024-11-22T23:55:05,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705208 2024-11-22T23:55:05,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705208 2024-11-22T23:55:05,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705208) roll requested 2024-11-22T23:55:05,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705217 2024-11-22T23:55:05,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705217 2024-11-22T23:55:05,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705217) roll requested 2024-11-22T23:55:05,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705225 2024-11-22T23:55:05,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705225 2024-11-22T23:55:05,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705225) roll requested 2024-11-22T23:55:05,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705242 2024-11-22T23:55:05,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705242 2024-11-22T23:55:05,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705242) roll requested 2024-11-22T23:55:05,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705250 2024-11-22T23:55:05,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705250 2024-11-22T23:55:05,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705250) roll requested 2024-11-22T23:55:05,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705258 2024-11-22T23:55:05,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705258 2024-11-22T23:55:05,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705258) roll requested 2024-11-22T23:55:05,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705267 2024-11-22T23:55:05,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705267 2024-11-22T23:55:05,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705267) roll requested 2024-11-22T23:55:05,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705277 2024-11-22T23:55:05,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705277 2024-11-22T23:55:05,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705277) roll requested 2024-11-22T23:55:05,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705285 2024-11-22T23:55:05,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705285 2024-11-22T23:55:05,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705285) roll requested 2024-11-22T23:55:05,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705292 2024-11-22T23:55:05,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705292 2024-11-22T23:55:05,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705292) roll requested 2024-11-22T23:55:05,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705299 2024-11-22T23:55:05,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705299 2024-11-22T23:55:05,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705299) roll requested 2024-11-22T23:55:05,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705309 2024-11-22T23:55:05,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705309 2024-11-22T23:55:05,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705309) roll requested 2024-11-22T23:55:05,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705316 2024-11-22T23:55:05,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705316 2024-11-22T23:55:05,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705316) roll requested 2024-11-22T23:55:05,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705322 2024-11-22T23:55:05,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705322 2024-11-22T23:55:05,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705322) roll requested 2024-11-22T23:55:05,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705329 2024-11-22T23:55:05,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705329 2024-11-22T23:55:05,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705329) roll requested 2024-11-22T23:55:05,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705337 2024-11-22T23:55:05,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705337 2024-11-22T23:55:05,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705337) roll requested 2024-11-22T23:55:05,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705343 2024-11-22T23:55:05,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705343 2024-11-22T23:55:05,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705343) roll requested 2024-11-22T23:55:05,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705350 2024-11-22T23:55:05,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705350 2024-11-22T23:55:05,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705350) roll requested 2024-11-22T23:55:05,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705358 2024-11-22T23:55:05,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705358 2024-11-22T23:55:05,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705358) roll requested 2024-11-22T23:55:05,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705370 2024-11-22T23:55:05,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705370 2024-11-22T23:55:05,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705370) roll requested 2024-11-22T23:55:05,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705377 2024-11-22T23:55:05,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705377 2024-11-22T23:55:05,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705377) roll requested 2024-11-22T23:55:05,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705383 2024-11-22T23:55:05,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705383 2024-11-22T23:55:05,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705383) roll requested 2024-11-22T23:55:05,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705391 2024-11-22T23:55:05,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705391 2024-11-22T23:55:05,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705391) roll requested 2024-11-22T23:55:05,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705398 2024-11-22T23:55:05,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705398 2024-11-22T23:55:05,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705398) roll requested 2024-11-22T23:55:05,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705406 2024-11-22T23:55:05,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705406 2024-11-22T23:55:05,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705406) roll requested 2024-11-22T23:55:05,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705417 2024-11-22T23:55:05,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705417 2024-11-22T23:55:05,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705417) roll requested 2024-11-22T23:55:05,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705424 2024-11-22T23:55:05,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705424 2024-11-22T23:55:05,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705424) roll requested 2024-11-22T23:55:05,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705431 2024-11-22T23:55:05,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705431 2024-11-22T23:55:05,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705431) roll requested 2024-11-22T23:55:05,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705438 2024-11-22T23:55:05,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705438 2024-11-22T23:55:05,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705438) roll requested 2024-11-22T23:55:05,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705445 2024-11-22T23:55:05,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705445 2024-11-22T23:55:05,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705445) roll requested 2024-11-22T23:55:05,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705461 2024-11-22T23:55:05,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705461 2024-11-22T23:55:05,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705461) roll requested 2024-11-22T23:55:05,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705471 2024-11-22T23:55:05,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705471 2024-11-22T23:55:05,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705471) roll requested 2024-11-22T23:55:05,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705478 2024-11-22T23:55:05,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705478 2024-11-22T23:55:05,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705478) roll requested 2024-11-22T23:55:05,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705486 2024-11-22T23:55:05,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705486 2024-11-22T23:55:05,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705486) roll requested 2024-11-22T23:55:05,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705492 2024-11-22T23:55:05,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705492 2024-11-22T23:55:05,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705492) roll requested 2024-11-22T23:55:05,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705500 2024-11-22T23:55:05,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705500 2024-11-22T23:55:05,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705500) roll requested 2024-11-22T23:55:05,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705506 2024-11-22T23:55:05,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705506 2024-11-22T23:55:05,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705506) roll requested 2024-11-22T23:55:05,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705513 2024-11-22T23:55:05,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705513 2024-11-22T23:55:05,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705513) roll requested 2024-11-22T23:55:05,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705520 2024-11-22T23:55:05,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705520 2024-11-22T23:55:05,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705520) roll requested 2024-11-22T23:55:05,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705527 2024-11-22T23:55:05,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705527 2024-11-22T23:55:05,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705527) roll requested 2024-11-22T23:55:05,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705534 2024-11-22T23:55:05,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705534 2024-11-22T23:55:05,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705534) roll requested 2024-11-22T23:55:05,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705541 2024-11-22T23:55:05,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705541 2024-11-22T23:55:05,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705541) roll requested 2024-11-22T23:55:05,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705549 2024-11-22T23:55:05,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705549 2024-11-22T23:55:05,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705549) roll requested 2024-11-22T23:55:05,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705556 2024-11-22T23:55:05,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705556 2024-11-22T23:55:05,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705556) roll requested 2024-11-22T23:55:05,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705563 2024-11-22T23:55:05,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705563 2024-11-22T23:55:05,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705563) roll requested 2024-11-22T23:55:05,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705571 2024-11-22T23:55:05,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705571 2024-11-22T23:55:05,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705571) roll requested 2024-11-22T23:55:05,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705578 2024-11-22T23:55:05,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705578 2024-11-22T23:55:05,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705578) roll requested 2024-11-22T23:55:05,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705586 2024-11-22T23:55:05,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705586 2024-11-22T23:55:05,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705586) roll requested 2024-11-22T23:55:05,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705593 2024-11-22T23:55:05,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705593 2024-11-22T23:55:05,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705593) roll requested 2024-11-22T23:55:05,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705602 2024-11-22T23:55:05,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705602 2024-11-22T23:55:05,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705602) roll requested 2024-11-22T23:55:05,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705609 2024-11-22T23:55:05,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705609 2024-11-22T23:55:05,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705609) roll requested 2024-11-22T23:55:05,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705617 2024-11-22T23:55:05,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705617 2024-11-22T23:55:05,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705617) roll requested 2024-11-22T23:55:05,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705626 2024-11-22T23:55:05,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705626 2024-11-22T23:55:05,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705626) roll requested 2024-11-22T23:55:05,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705633 2024-11-22T23:55:05,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705633 2024-11-22T23:55:05,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705633) roll requested 2024-11-22T23:55:05,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705640 2024-11-22T23:55:05,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705640 2024-11-22T23:55:05,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705640) roll requested 2024-11-22T23:55:05,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705647 2024-11-22T23:55:05,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705647 2024-11-22T23:55:05,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705647) roll requested 2024-11-22T23:55:05,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705655 2024-11-22T23:55:05,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705655 2024-11-22T23:55:05,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705655) roll requested 2024-11-22T23:55:05,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705662 2024-11-22T23:55:05,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705662 2024-11-22T23:55:05,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705662) roll requested 2024-11-22T23:55:05,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705669 2024-11-22T23:55:05,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705669 2024-11-22T23:55:05,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705669) roll requested 2024-11-22T23:55:05,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705676 2024-11-22T23:55:05,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705676 2024-11-22T23:55:05,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705676) roll requested 2024-11-22T23:55:05,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705684 2024-11-22T23:55:05,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705684 2024-11-22T23:55:05,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705684) roll requested 2024-11-22T23:55:05,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705691 2024-11-22T23:55:05,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705691 2024-11-22T23:55:05,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705691) roll requested 2024-11-22T23:55:05,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705699 2024-11-22T23:55:05,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705699 2024-11-22T23:55:05,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705699) roll requested 2024-11-22T23:55:05,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705706 2024-11-22T23:55:05,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705706 2024-11-22T23:55:05,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705706) roll requested 2024-11-22T23:55:05,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705713 2024-11-22T23:55:05,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705713 2024-11-22T23:55:05,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705713) roll requested 2024-11-22T23:55:05,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705720 2024-11-22T23:55:05,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705720 2024-11-22T23:55:05,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705720) roll requested 2024-11-22T23:55:05,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705728 2024-11-22T23:55:05,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705728 2024-11-22T23:55:05,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705728) roll requested 2024-11-22T23:55:05,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705735 2024-11-22T23:55:05,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705735 2024-11-22T23:55:05,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705735) roll requested 2024-11-22T23:55:05,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705742 2024-11-22T23:55:05,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705742 2024-11-22T23:55:05,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705742) roll requested 2024-11-22T23:55:05,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705749 2024-11-22T23:55:05,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705749 2024-11-22T23:55:05,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705749) roll requested 2024-11-22T23:55:05,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705757 2024-11-22T23:55:05,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705757 2024-11-22T23:55:05,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705757) roll requested 2024-11-22T23:55:05,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705764 2024-11-22T23:55:05,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705764 2024-11-22T23:55:05,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705764) roll requested 2024-11-22T23:55:05,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705771 2024-11-22T23:55:05,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705771 2024-11-22T23:55:05,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705771) roll requested 2024-11-22T23:55:05,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705778 2024-11-22T23:55:05,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705778 2024-11-22T23:55:05,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705778) roll requested 2024-11-22T23:55:05,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705785 2024-11-22T23:55:05,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705785 2024-11-22T23:55:05,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705785) roll requested 2024-11-22T23:55:05,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705791 2024-11-22T23:55:05,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705791 2024-11-22T23:55:05,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705791) roll requested 2024-11-22T23:55:05,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705799 2024-11-22T23:55:05,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705799 2024-11-22T23:55:05,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705799) roll requested 2024-11-22T23:55:05,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705806 2024-11-22T23:55:05,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705806 2024-11-22T23:55:05,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705806) roll requested 2024-11-22T23:55:05,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705812 2024-11-22T23:55:05,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705812 2024-11-22T23:55:05,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705812) roll requested 2024-11-22T23:55:05,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705820 2024-11-22T23:55:05,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705820 2024-11-22T23:55:05,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705820) roll requested 2024-11-22T23:55:05,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705828 2024-11-22T23:55:05,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705828 2024-11-22T23:55:05,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705828) roll requested 2024-11-22T23:55:05,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705835 2024-11-22T23:55:05,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705835 2024-11-22T23:55:05,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705835) roll requested 2024-11-22T23:55:05,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705842 2024-11-22T23:55:05,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705842 2024-11-22T23:55:05,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705842) roll requested 2024-11-22T23:55:05,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705849 2024-11-22T23:55:05,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705849 2024-11-22T23:55:05,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705849) roll requested 2024-11-22T23:55:05,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705857 2024-11-22T23:55:05,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705857 2024-11-22T23:55:05,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705857) roll requested 2024-11-22T23:55:05,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705864 2024-11-22T23:55:05,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705864 2024-11-22T23:55:05,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705864) roll requested 2024-11-22T23:55:05,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705871 2024-11-22T23:55:05,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705871 2024-11-22T23:55:05,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705871) roll requested 2024-11-22T23:55:05,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705878 2024-11-22T23:55:05,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705878 2024-11-22T23:55:05,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705878) roll requested 2024-11-22T23:55:05,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705885 2024-11-22T23:55:05,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705885 2024-11-22T23:55:05,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705885) roll requested 2024-11-22T23:55:05,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705892 2024-11-22T23:55:05,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705892 2024-11-22T23:55:05,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705892) roll requested 2024-11-22T23:55:05,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705899 2024-11-22T23:55:05,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705899 2024-11-22T23:55:05,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705899) roll requested 2024-11-22T23:55:05,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705907 2024-11-22T23:55:05,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705907 2024-11-22T23:55:05,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705907) roll requested 2024-11-22T23:55:05,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705914 2024-11-22T23:55:05,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705914 2024-11-22T23:55:05,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705914) roll requested 2024-11-22T23:55:05,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705922 2024-11-22T23:55:05,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705922 2024-11-22T23:55:05,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705922) roll requested 2024-11-22T23:55:05,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705929 2024-11-22T23:55:05,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705929 2024-11-22T23:55:05,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705929) roll requested 2024-11-22T23:55:05,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705936 2024-11-22T23:55:05,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705936 2024-11-22T23:55:05,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705936) roll requested 2024-11-22T23:55:05,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705944 2024-11-22T23:55:05,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705944 2024-11-22T23:55:05,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705944) roll requested 2024-11-22T23:55:05,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705950 2024-11-22T23:55:05,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705950 2024-11-22T23:55:05,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705950) roll requested 2024-11-22T23:55:05,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705958 2024-11-22T23:55:05,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705958 2024-11-22T23:55:05,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705958) roll requested 2024-11-22T23:55:05,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705965 2024-11-22T23:55:05,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705965 2024-11-22T23:55:05,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705965) roll requested 2024-11-22T23:55:05,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705972 2024-11-22T23:55:05,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705972 2024-11-22T23:55:05,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705972) roll requested 2024-11-22T23:55:05,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705979 2024-11-22T23:55:05,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705979 2024-11-22T23:55:05,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705979) roll requested 2024-11-22T23:55:05,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705986 2024-11-22T23:55:05,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705986 2024-11-22T23:55:05,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:05,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:05,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705986) roll requested 2024-11-22T23:55:05,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319705993 2024-11-22T23:55:05,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:05,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705993 2024-11-22T23:55:06,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319705993) roll requested 2024-11-22T23:55:06,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706000 2024-11-22T23:55:06,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319705993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706000 2024-11-22T23:55:06,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706000) roll requested 2024-11-22T23:55:06,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706011 2024-11-22T23:55:06,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706011 2024-11-22T23:55:06,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706011) roll requested 2024-11-22T23:55:06,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706020 2024-11-22T23:55:06,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706020 2024-11-22T23:55:06,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706020) roll requested 2024-11-22T23:55:06,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706027 2024-11-22T23:55:06,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706027 2024-11-22T23:55:06,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706027) roll requested 2024-11-22T23:55:06,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706034 2024-11-22T23:55:06,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706034 2024-11-22T23:55:06,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706034) roll requested 2024-11-22T23:55:06,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706042 2024-11-22T23:55:06,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706042 2024-11-22T23:55:06,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706042) roll requested 2024-11-22T23:55:06,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706049 2024-11-22T23:55:06,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706049 2024-11-22T23:55:06,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706049) roll requested 2024-11-22T23:55:06,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706056 2024-11-22T23:55:06,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706056 2024-11-22T23:55:06,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706056) roll requested 2024-11-22T23:55:06,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706063 2024-11-22T23:55:06,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706063 2024-11-22T23:55:06,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706063) roll requested 2024-11-22T23:55:06,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706073 2024-11-22T23:55:06,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706073 2024-11-22T23:55:06,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706073) roll requested 2024-11-22T23:55:06,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706081 2024-11-22T23:55:06,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706081 2024-11-22T23:55:06,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706081) roll requested 2024-11-22T23:55:06,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706100 2024-11-22T23:55:06,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706100 2024-11-22T23:55:06,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706100) roll requested 2024-11-22T23:55:06,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706107 2024-11-22T23:55:06,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706107 2024-11-22T23:55:06,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706107) roll requested 2024-11-22T23:55:06,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706115 2024-11-22T23:55:06,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706115 2024-11-22T23:55:06,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706115) roll requested 2024-11-22T23:55:06,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706122 2024-11-22T23:55:06,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706122 2024-11-22T23:55:06,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706122) roll requested 2024-11-22T23:55:06,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706145 2024-11-22T23:55:06,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706145 2024-11-22T23:55:06,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706145) roll requested 2024-11-22T23:55:06,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706152 2024-11-22T23:55:06,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706152 2024-11-22T23:55:06,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706152) roll requested 2024-11-22T23:55:06,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706160 2024-11-22T23:55:06,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706160 2024-11-22T23:55:06,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706160) roll requested 2024-11-22T23:55:06,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706169 2024-11-22T23:55:06,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706169 2024-11-22T23:55:06,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706169) roll requested 2024-11-22T23:55:06,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706175 2024-11-22T23:55:06,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706175 2024-11-22T23:55:06,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706175) roll requested 2024-11-22T23:55:06,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706185 2024-11-22T23:55:06,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706185 2024-11-22T23:55:06,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706185) roll requested 2024-11-22T23:55:06,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706196 2024-11-22T23:55:06,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706196 2024-11-22T23:55:06,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706196) roll requested 2024-11-22T23:55:06,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706204 2024-11-22T23:55:06,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706204 2024-11-22T23:55:06,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706204) roll requested 2024-11-22T23:55:06,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706212 2024-11-22T23:55:06,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706212 2024-11-22T23:55:06,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706212) roll requested 2024-11-22T23:55:06,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706219 2024-11-22T23:55:06,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706219 2024-11-22T23:55:06,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706219) roll requested 2024-11-22T23:55:06,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706227 2024-11-22T23:55:06,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706227 2024-11-22T23:55:06,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706227) roll requested 2024-11-22T23:55:06,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706234 2024-11-22T23:55:06,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706234 2024-11-22T23:55:06,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706234) roll requested 2024-11-22T23:55:06,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706241 2024-11-22T23:55:06,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706241 2024-11-22T23:55:06,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706241) roll requested 2024-11-22T23:55:06,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706248 2024-11-22T23:55:06,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706248 2024-11-22T23:55:06,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706248) roll requested 2024-11-22T23:55:06,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706255 2024-11-22T23:55:06,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706255 2024-11-22T23:55:06,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706255) roll requested 2024-11-22T23:55:06,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706262 2024-11-22T23:55:06,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706262 2024-11-22T23:55:06,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706262) roll requested 2024-11-22T23:55:06,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706269 2024-11-22T23:55:06,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706269 2024-11-22T23:55:06,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706269) roll requested 2024-11-22T23:55:06,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706276 2024-11-22T23:55:06,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706276 2024-11-22T23:55:06,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706276) roll requested 2024-11-22T23:55:06,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706283 2024-11-22T23:55:06,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706283 2024-11-22T23:55:06,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706283) roll requested 2024-11-22T23:55:06,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706291 2024-11-22T23:55:06,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706291 2024-11-22T23:55:06,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706291) roll requested 2024-11-22T23:55:06,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706299 2024-11-22T23:55:06,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706299 2024-11-22T23:55:06,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706299) roll requested 2024-11-22T23:55:06,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706306 2024-11-22T23:55:06,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706306 2024-11-22T23:55:06,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706306) roll requested 2024-11-22T23:55:06,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706313 2024-11-22T23:55:06,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706313 2024-11-22T23:55:06,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706313) roll requested 2024-11-22T23:55:06,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706321 2024-11-22T23:55:06,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706321 2024-11-22T23:55:06,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706321) roll requested 2024-11-22T23:55:06,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706329 2024-11-22T23:55:06,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706329 2024-11-22T23:55:06,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706329) roll requested 2024-11-22T23:55:06,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706336 2024-11-22T23:55:06,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706336 2024-11-22T23:55:06,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706336) roll requested 2024-11-22T23:55:06,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706342 2024-11-22T23:55:06,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706342 2024-11-22T23:55:06,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706342) roll requested 2024-11-22T23:55:06,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706349 2024-11-22T23:55:06,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706349 2024-11-22T23:55:06,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706349) roll requested 2024-11-22T23:55:06,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706356 2024-11-22T23:55:06,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706356 2024-11-22T23:55:06,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706356) roll requested 2024-11-22T23:55:06,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706363 2024-11-22T23:55:06,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706363 2024-11-22T23:55:06,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706363) roll requested 2024-11-22T23:55:06,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706370 2024-11-22T23:55:06,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706370 2024-11-22T23:55:06,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706370) roll requested 2024-11-22T23:55:06,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706377 2024-11-22T23:55:06,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706377 2024-11-22T23:55:06,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706377) roll requested 2024-11-22T23:55:06,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706384 2024-11-22T23:55:06,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706384 2024-11-22T23:55:06,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706384) roll requested 2024-11-22T23:55:06,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706391 2024-11-22T23:55:06,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706391 2024-11-22T23:55:06,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706391) roll requested 2024-11-22T23:55:06,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706399 2024-11-22T23:55:06,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706399 2024-11-22T23:55:06,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706399) roll requested 2024-11-22T23:55:06,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706405 2024-11-22T23:55:06,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706405 2024-11-22T23:55:06,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706405) roll requested 2024-11-22T23:55:06,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706412 2024-11-22T23:55:06,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706412 2024-11-22T23:55:06,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,419 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706412) roll requested 2024-11-22T23:55:06,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706419 2024-11-22T23:55:06,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706419 2024-11-22T23:55:06,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706419) roll requested 2024-11-22T23:55:06,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706426 2024-11-22T23:55:06,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706426 2024-11-22T23:55:06,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706426) roll requested 2024-11-22T23:55:06,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706433 2024-11-22T23:55:06,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706433 2024-11-22T23:55:06,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706433) roll requested 2024-11-22T23:55:06,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706439 2024-11-22T23:55:06,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706439 2024-11-22T23:55:06,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706439) roll requested 2024-11-22T23:55:06,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706447 2024-11-22T23:55:06,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706447 2024-11-22T23:55:06,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706447) roll requested 2024-11-22T23:55:06,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706454 2024-11-22T23:55:06,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706454 2024-11-22T23:55:06,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706454) roll requested 2024-11-22T23:55:06,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706461 2024-11-22T23:55:06,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706461 2024-11-22T23:55:06,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706461) roll requested 2024-11-22T23:55:06,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706468 2024-11-22T23:55:06,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706468 2024-11-22T23:55:06,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706468) roll requested 2024-11-22T23:55:06,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706481 2024-11-22T23:55:06,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706481 2024-11-22T23:55:06,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706481) roll requested 2024-11-22T23:55:06,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706493 2024-11-22T23:55:06,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706493 2024-11-22T23:55:06,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706493) roll requested 2024-11-22T23:55:06,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706500 2024-11-22T23:55:06,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706500 2024-11-22T23:55:06,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706500) roll requested 2024-11-22T23:55:06,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706507 2024-11-22T23:55:06,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706507 2024-11-22T23:55:06,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706507) roll requested 2024-11-22T23:55:06,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706515 2024-11-22T23:55:06,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706515 2024-11-22T23:55:06,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706515) roll requested 2024-11-22T23:55:06,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706522 2024-11-22T23:55:06,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706522 2024-11-22T23:55:06,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706522) roll requested 2024-11-22T23:55:06,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706530 2024-11-22T23:55:06,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706530 2024-11-22T23:55:06,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706530) roll requested 2024-11-22T23:55:06,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706537 2024-11-22T23:55:06,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706537 2024-11-22T23:55:06,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706537) roll requested 2024-11-22T23:55:06,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706544 2024-11-22T23:55:06,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706544 2024-11-22T23:55:06,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706544) roll requested 2024-11-22T23:55:06,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706551 2024-11-22T23:55:06,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706551 2024-11-22T23:55:06,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706551) roll requested 2024-11-22T23:55:06,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706558 2024-11-22T23:55:06,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706558 2024-11-22T23:55:06,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706558) roll requested 2024-11-22T23:55:06,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706566 2024-11-22T23:55:06,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706566 2024-11-22T23:55:06,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706566) roll requested 2024-11-22T23:55:06,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706579 2024-11-22T23:55:06,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706579 2024-11-22T23:55:06,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706579) roll requested 2024-11-22T23:55:06,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706586 2024-11-22T23:55:06,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706586 2024-11-22T23:55:06,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706586) roll requested 2024-11-22T23:55:06,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706593 2024-11-22T23:55:06,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706593 2024-11-22T23:55:06,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706593) roll requested 2024-11-22T23:55:06,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706600 2024-11-22T23:55:06,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706600 2024-11-22T23:55:06,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706600) roll requested 2024-11-22T23:55:06,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706607 2024-11-22T23:55:06,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706607 2024-11-22T23:55:06,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706607) roll requested 2024-11-22T23:55:06,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706614 2024-11-22T23:55:06,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706614 2024-11-22T23:55:06,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706614) roll requested 2024-11-22T23:55:06,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706622 2024-11-22T23:55:06,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706622 2024-11-22T23:55:06,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706622) roll requested 2024-11-22T23:55:06,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706629 2024-11-22T23:55:06,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706629 2024-11-22T23:55:06,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706629) roll requested 2024-11-22T23:55:06,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706636 2024-11-22T23:55:06,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706636 2024-11-22T23:55:06,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706636) roll requested 2024-11-22T23:55:06,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706643 2024-11-22T23:55:06,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706643 2024-11-22T23:55:06,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706643) roll requested 2024-11-22T23:55:06,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706650 2024-11-22T23:55:06,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706650 2024-11-22T23:55:06,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706650) roll requested 2024-11-22T23:55:06,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706657 2024-11-22T23:55:06,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706657 2024-11-22T23:55:06,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706657) roll requested 2024-11-22T23:55:06,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706664 2024-11-22T23:55:06,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706664 2024-11-22T23:55:06,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706664) roll requested 2024-11-22T23:55:06,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706671 2024-11-22T23:55:06,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706671 2024-11-22T23:55:06,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706671) roll requested 2024-11-22T23:55:06,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706677 2024-11-22T23:55:06,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706677 2024-11-22T23:55:06,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706677) roll requested 2024-11-22T23:55:06,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706684 2024-11-22T23:55:06,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706684 2024-11-22T23:55:06,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706684) roll requested 2024-11-22T23:55:06,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706691 2024-11-22T23:55:06,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706691 2024-11-22T23:55:06,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706691) roll requested 2024-11-22T23:55:06,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706698 2024-11-22T23:55:06,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706698 2024-11-22T23:55:06,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706698) roll requested 2024-11-22T23:55:06,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706705 2024-11-22T23:55:06,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706705 2024-11-22T23:55:06,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706705) roll requested 2024-11-22T23:55:06,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706712 2024-11-22T23:55:06,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706712 2024-11-22T23:55:06,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706712) roll requested 2024-11-22T23:55:06,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706719 2024-11-22T23:55:06,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706719 2024-11-22T23:55:06,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706719) roll requested 2024-11-22T23:55:06,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706727 2024-11-22T23:55:06,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706727 2024-11-22T23:55:06,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706727) roll requested 2024-11-22T23:55:06,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706734 2024-11-22T23:55:06,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706734 2024-11-22T23:55:06,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706734) roll requested 2024-11-22T23:55:06,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706744 2024-11-22T23:55:06,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706744 2024-11-22T23:55:06,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706744) roll requested 2024-11-22T23:55:06,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706752 2024-11-22T23:55:06,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706752 2024-11-22T23:55:06,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706752) roll requested 2024-11-22T23:55:06,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706762 2024-11-22T23:55:06,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706762 2024-11-22T23:55:06,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706762) roll requested 2024-11-22T23:55:06,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706768 2024-11-22T23:55:06,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706768 2024-11-22T23:55:06,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706768) roll requested 2024-11-22T23:55:06,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706775 2024-11-22T23:55:06,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706775 2024-11-22T23:55:06,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706775) roll requested 2024-11-22T23:55:06,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706781 2024-11-22T23:55:06,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706781 2024-11-22T23:55:06,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706781) roll requested 2024-11-22T23:55:06,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706787 2024-11-22T23:55:06,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706787 2024-11-22T23:55:06,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706787) roll requested 2024-11-22T23:55:06,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706794 2024-11-22T23:55:06,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706794 2024-11-22T23:55:06,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706794) roll requested 2024-11-22T23:55:06,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706800 2024-11-22T23:55:06,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706800 2024-11-22T23:55:06,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706800) roll requested 2024-11-22T23:55:06,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706812 2024-11-22T23:55:06,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706812 2024-11-22T23:55:06,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706812) roll requested 2024-11-22T23:55:06,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706818 2024-11-22T23:55:06,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706818 2024-11-22T23:55:06,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706818) roll requested 2024-11-22T23:55:06,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706825 2024-11-22T23:55:06,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706825 2024-11-22T23:55:06,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706825) roll requested 2024-11-22T23:55:06,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706831 2024-11-22T23:55:06,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706831 2024-11-22T23:55:06,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706831) roll requested 2024-11-22T23:55:06,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706837 2024-11-22T23:55:06,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706837 2024-11-22T23:55:06,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706837) roll requested 2024-11-22T23:55:06,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706844 2024-11-22T23:55:06,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706844 2024-11-22T23:55:06,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706844) roll requested 2024-11-22T23:55:06,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706850 2024-11-22T23:55:06,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706850 2024-11-22T23:55:06,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706850) roll requested 2024-11-22T23:55:06,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706856 2024-11-22T23:55:06,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706856 2024-11-22T23:55:06,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706856) roll requested 2024-11-22T23:55:06,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706862 2024-11-22T23:55:06,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706862 2024-11-22T23:55:06,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706862) roll requested 2024-11-22T23:55:06,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706869 2024-11-22T23:55:06,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706869 2024-11-22T23:55:06,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706869) roll requested 2024-11-22T23:55:06,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706875 2024-11-22T23:55:06,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706875 2024-11-22T23:55:06,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706875) roll requested 2024-11-22T23:55:06,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706881 2024-11-22T23:55:06,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706881 2024-11-22T23:55:06,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706881) roll requested 2024-11-22T23:55:06,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706888 2024-11-22T23:55:06,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706888 2024-11-22T23:55:06,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706888) roll requested 2024-11-22T23:55:06,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706894 2024-11-22T23:55:06,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706894 2024-11-22T23:55:06,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706894) roll requested 2024-11-22T23:55:06,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706900 2024-11-22T23:55:06,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706900 2024-11-22T23:55:06,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706900) roll requested 2024-11-22T23:55:06,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706907 2024-11-22T23:55:06,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706907 2024-11-22T23:55:06,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706907) roll requested 2024-11-22T23:55:06,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706913 2024-11-22T23:55:06,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706913 2024-11-22T23:55:06,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706913) roll requested 2024-11-22T23:55:06,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706919 2024-11-22T23:55:06,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706919 2024-11-22T23:55:06,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706919) roll requested 2024-11-22T23:55:06,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706925 2024-11-22T23:55:06,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706925 2024-11-22T23:55:06,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706925) roll requested 2024-11-22T23:55:06,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706931 2024-11-22T23:55:06,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706931 2024-11-22T23:55:06,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706931) roll requested 2024-11-22T23:55:06,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706938 2024-11-22T23:55:06,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706938 2024-11-22T23:55:06,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706938) roll requested 2024-11-22T23:55:06,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706944 2024-11-22T23:55:06,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706944 2024-11-22T23:55:06,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706944) roll requested 2024-11-22T23:55:06,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706950 2024-11-22T23:55:06,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706950 2024-11-22T23:55:06,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706950) roll requested 2024-11-22T23:55:06,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706956 2024-11-22T23:55:06,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706956 2024-11-22T23:55:06,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706956) roll requested 2024-11-22T23:55:06,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706963 2024-11-22T23:55:06,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706963 2024-11-22T23:55:06,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706963) roll requested 2024-11-22T23:55:06,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706970 2024-11-22T23:55:06,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706970 2024-11-22T23:55:06,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706970) roll requested 2024-11-22T23:55:06,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706976 2024-11-22T23:55:06,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706976 2024-11-22T23:55:06,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706976) roll requested 2024-11-22T23:55:06,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706983 2024-11-22T23:55:06,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706983 2024-11-22T23:55:06,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706983) roll requested 2024-11-22T23:55:06,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706992 2024-11-22T23:55:06,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:06,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706992 2024-11-22T23:55:06,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:06,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:06,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706992) roll requested 2024-11-22T23:55:06,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319706998 2024-11-22T23:55:07,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706998 2024-11-22T23:55:07,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319706998) roll requested 2024-11-22T23:55:07,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707005 2024-11-22T23:55:07,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319706998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707005 2024-11-22T23:55:07,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707005) roll requested 2024-11-22T23:55:07,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707011 2024-11-22T23:55:07,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707011 2024-11-22T23:55:07,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707011) roll requested 2024-11-22T23:55:07,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707017 2024-11-22T23:55:07,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707017 2024-11-22T23:55:07,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707017) roll requested 2024-11-22T23:55:07,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707025 2024-11-22T23:55:07,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707025 2024-11-22T23:55:07,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707025) roll requested 2024-11-22T23:55:07,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707031 2024-11-22T23:55:07,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707031 2024-11-22T23:55:07,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707031) roll requested 2024-11-22T23:55:07,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707037 2024-11-22T23:55:07,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707037 2024-11-22T23:55:07,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707037) roll requested 2024-11-22T23:55:07,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707044 2024-11-22T23:55:07,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707044 2024-11-22T23:55:07,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707044) roll requested 2024-11-22T23:55:07,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707050 2024-11-22T23:55:07,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707050 2024-11-22T23:55:07,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707050) roll requested 2024-11-22T23:55:07,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707056 2024-11-22T23:55:07,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707056 2024-11-22T23:55:07,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707056) roll requested 2024-11-22T23:55:07,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707081 2024-11-22T23:55:07,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707081 2024-11-22T23:55:07,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707081) roll requested 2024-11-22T23:55:07,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707092 2024-11-22T23:55:07,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707092 2024-11-22T23:55:07,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707092) roll requested 2024-11-22T23:55:07,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707099 2024-11-22T23:55:07,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707099 2024-11-22T23:55:07,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707099) roll requested 2024-11-22T23:55:07,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707105 2024-11-22T23:55:07,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707105 2024-11-22T23:55:07,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707105) roll requested 2024-11-22T23:55:07,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707111 2024-11-22T23:55:07,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707111 2024-11-22T23:55:07,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707111) roll requested 2024-11-22T23:55:07,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707118 2024-11-22T23:55:07,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707118 2024-11-22T23:55:07,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707118) roll requested 2024-11-22T23:55:07,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707124 2024-11-22T23:55:07,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707124 2024-11-22T23:55:07,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707124) roll requested 2024-11-22T23:55:07,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707130 2024-11-22T23:55:07,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707130 2024-11-22T23:55:07,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707130) roll requested 2024-11-22T23:55:07,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707136 2024-11-22T23:55:07,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707136 2024-11-22T23:55:07,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707136) roll requested 2024-11-22T23:55:07,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707143 2024-11-22T23:55:07,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707143 2024-11-22T23:55:07,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707143) roll requested 2024-11-22T23:55:07,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707149 2024-11-22T23:55:07,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707149 2024-11-22T23:55:07,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707149) roll requested 2024-11-22T23:55:07,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707155 2024-11-22T23:55:07,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707155 2024-11-22T23:55:07,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707155) roll requested 2024-11-22T23:55:07,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707162 2024-11-22T23:55:07,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707162 2024-11-22T23:55:07,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707162) roll requested 2024-11-22T23:55:07,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707169 2024-11-22T23:55:07,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707169 2024-11-22T23:55:07,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707169) roll requested 2024-11-22T23:55:07,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707175 2024-11-22T23:55:07,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707175 2024-11-22T23:55:07,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707175) roll requested 2024-11-22T23:55:07,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707181 2024-11-22T23:55:07,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707181 2024-11-22T23:55:07,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707181) roll requested 2024-11-22T23:55:07,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707187 2024-11-22T23:55:07,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707187 2024-11-22T23:55:07,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707187) roll requested 2024-11-22T23:55:07,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707195 2024-11-22T23:55:07,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707195 2024-11-22T23:55:07,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707195) roll requested 2024-11-22T23:55:07,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707201 2024-11-22T23:55:07,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707201 2024-11-22T23:55:07,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707201) roll requested 2024-11-22T23:55:07,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707207 2024-11-22T23:55:07,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707207 2024-11-22T23:55:07,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707207) roll requested 2024-11-22T23:55:07,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707219 2024-11-22T23:55:07,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707219 2024-11-22T23:55:07,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707219) roll requested 2024-11-22T23:55:07,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707228 2024-11-22T23:55:07,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707228 2024-11-22T23:55:07,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707228) roll requested 2024-11-22T23:55:07,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707237 2024-11-22T23:55:07,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707237 2024-11-22T23:55:07,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707237) roll requested 2024-11-22T23:55:07,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707249 2024-11-22T23:55:07,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707249 2024-11-22T23:55:07,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707249) roll requested 2024-11-22T23:55:07,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707257 2024-11-22T23:55:07,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707257 2024-11-22T23:55:07,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707257) roll requested 2024-11-22T23:55:07,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707263 2024-11-22T23:55:07,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707263 2024-11-22T23:55:07,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707263) roll requested 2024-11-22T23:55:07,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707271 2024-11-22T23:55:07,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707271 2024-11-22T23:55:07,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707271) roll requested 2024-11-22T23:55:07,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707281 2024-11-22T23:55:07,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707271 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707281 2024-11-22T23:55:07,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707281) roll requested 2024-11-22T23:55:07,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707288 2024-11-22T23:55:07,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707288 2024-11-22T23:55:07,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707288) roll requested 2024-11-22T23:55:07,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707294 2024-11-22T23:55:07,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707294 2024-11-22T23:55:07,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707294) roll requested 2024-11-22T23:55:07,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707301 2024-11-22T23:55:07,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707301 2024-11-22T23:55:07,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707301) roll requested 2024-11-22T23:55:07,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707307 2024-11-22T23:55:07,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707307 2024-11-22T23:55:07,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707307) roll requested 2024-11-22T23:55:07,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707313 2024-11-22T23:55:07,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707313 2024-11-22T23:55:07,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707313) roll requested 2024-11-22T23:55:07,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707320 2024-11-22T23:55:07,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707320 2024-11-22T23:55:07,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707320) roll requested 2024-11-22T23:55:07,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707327 2024-11-22T23:55:07,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707327 2024-11-22T23:55:07,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707327) roll requested 2024-11-22T23:55:07,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707333 2024-11-22T23:55:07,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707333 2024-11-22T23:55:07,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707333) roll requested 2024-11-22T23:55:07,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707340 2024-11-22T23:55:07,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707340 2024-11-22T23:55:07,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707340) roll requested 2024-11-22T23:55:07,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707346 2024-11-22T23:55:07,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707346 2024-11-22T23:55:07,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707340 is not closed yet, will try archiving it next time 2024-11-22T23:55:07,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707346) roll requested 2024-11-22T23:55:07,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707353 2024-11-22T23:55:07,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707353 2024-11-22T23:55:07,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707353) roll requested 2024-11-22T23:55:07,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707361 2024-11-22T23:55:07,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707361 2024-11-22T23:55:07,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707361) roll requested 2024-11-22T23:55:07,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707367 2024-11-22T23:55:07,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707367 2024-11-22T23:55:07,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707367) roll requested 2024-11-22T23:55:07,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707374 2024-11-22T23:55:07,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707374 2024-11-22T23:55:07,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707374) roll requested 2024-11-22T23:55:07,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707381 2024-11-22T23:55:07,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707381 2024-11-22T23:55:07,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707381) roll requested 2024-11-22T23:55:07,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707390 2024-11-22T23:55:07,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707390 2024-11-22T23:55:07,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707390) roll requested 2024-11-22T23:55:07,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707397 2024-11-22T23:55:07,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707397 2024-11-22T23:55:07,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707397) roll requested 2024-11-22T23:55:07,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707404 2024-11-22T23:55:07,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707404 2024-11-22T23:55:07,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707404) roll requested 2024-11-22T23:55:07,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707411 2024-11-22T23:55:07,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707411 2024-11-22T23:55:07,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707411) roll requested 2024-11-22T23:55:07,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707418 2024-11-22T23:55:07,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707418 2024-11-22T23:55:07,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707418) roll requested 2024-11-22T23:55:07,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707425 2024-11-22T23:55:07,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707425 2024-11-22T23:55:07,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707425) roll requested 2024-11-22T23:55:07,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707432 2024-11-22T23:55:07,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707432 2024-11-22T23:55:07,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707432) roll requested 2024-11-22T23:55:07,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707449 2024-11-22T23:55:07,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707449 2024-11-22T23:55:07,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707449) roll requested 2024-11-22T23:55:07,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707456 2024-11-22T23:55:07,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707456 2024-11-22T23:55:07,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707456) roll requested 2024-11-22T23:55:07,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707464 2024-11-22T23:55:07,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707464 2024-11-22T23:55:07,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707464) roll requested 2024-11-22T23:55:07,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707471 2024-11-22T23:55:07,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707471 2024-11-22T23:55:07,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707471) roll requested 2024-11-22T23:55:07,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707478 2024-11-22T23:55:07,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707478 2024-11-22T23:55:07,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707478) roll requested 2024-11-22T23:55:07,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707484 2024-11-22T23:55:07,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707484 2024-11-22T23:55:07,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707484) roll requested 2024-11-22T23:55:07,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707491 2024-11-22T23:55:07,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707491 2024-11-22T23:55:07,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707491) roll requested 2024-11-22T23:55:07,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707497 2024-11-22T23:55:07,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707497 2024-11-22T23:55:07,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707497) roll requested 2024-11-22T23:55:07,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707504 2024-11-22T23:55:07,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707504 2024-11-22T23:55:07,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707504) roll requested 2024-11-22T23:55:07,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707511 2024-11-22T23:55:07,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707511 2024-11-22T23:55:07,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707511) roll requested 2024-11-22T23:55:07,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707518 2024-11-22T23:55:07,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707518 2024-11-22T23:55:07,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707518) roll requested 2024-11-22T23:55:07,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707525 2024-11-22T23:55:07,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707525 2024-11-22T23:55:07,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707525) roll requested 2024-11-22T23:55:07,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707532 2024-11-22T23:55:07,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707532 2024-11-22T23:55:07,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707532) roll requested 2024-11-22T23:55:07,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707539 2024-11-22T23:55:07,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707539 2024-11-22T23:55:07,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707539) roll requested 2024-11-22T23:55:07,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707546 2024-11-22T23:55:07,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707546 2024-11-22T23:55:07,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707546) roll requested 2024-11-22T23:55:07,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707553 2024-11-22T23:55:07,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707553 2024-11-22T23:55:07,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707553) roll requested 2024-11-22T23:55:07,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707560 2024-11-22T23:55:07,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707560 2024-11-22T23:55:07,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707560) roll requested 2024-11-22T23:55:07,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707567 2024-11-22T23:55:07,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707567 2024-11-22T23:55:07,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707567) roll requested 2024-11-22T23:55:07,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707574 2024-11-22T23:55:07,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707574 2024-11-22T23:55:07,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707574) roll requested 2024-11-22T23:55:07,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707581 2024-11-22T23:55:07,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707581 2024-11-22T23:55:07,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707581) roll requested 2024-11-22T23:55:07,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707588 2024-11-22T23:55:07,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707588 2024-11-22T23:55:07,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707588) roll requested 2024-11-22T23:55:07,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707594 2024-11-22T23:55:07,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707594 2024-11-22T23:55:07,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707594) roll requested 2024-11-22T23:55:07,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707605 2024-11-22T23:55:07,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707605 2024-11-22T23:55:07,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707605) roll requested 2024-11-22T23:55:07,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707612 2024-11-22T23:55:07,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707612 2024-11-22T23:55:07,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707612) roll requested 2024-11-22T23:55:07,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707619 2024-11-22T23:55:07,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707619 2024-11-22T23:55:07,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707619) roll requested 2024-11-22T23:55:07,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707626 2024-11-22T23:55:07,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707626 2024-11-22T23:55:07,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707626) roll requested 2024-11-22T23:55:07,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707632 2024-11-22T23:55:07,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707632 2024-11-22T23:55:07,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707632) roll requested 2024-11-22T23:55:07,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707639 2024-11-22T23:55:07,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707639 2024-11-22T23:55:07,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707639) roll requested 2024-11-22T23:55:07,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707646 2024-11-22T23:55:07,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707646 2024-11-22T23:55:07,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707646) roll requested 2024-11-22T23:55:07,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707653 2024-11-22T23:55:07,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707653 2024-11-22T23:55:07,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707653) roll requested 2024-11-22T23:55:07,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707660 2024-11-22T23:55:07,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707660 2024-11-22T23:55:07,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707660) roll requested 2024-11-22T23:55:07,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707666 2024-11-22T23:55:07,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707666 2024-11-22T23:55:07,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707666) roll requested 2024-11-22T23:55:07,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707673 2024-11-22T23:55:07,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707673 2024-11-22T23:55:07,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707673) roll requested 2024-11-22T23:55:07,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707679 2024-11-22T23:55:07,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707679 2024-11-22T23:55:07,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707679) roll requested 2024-11-22T23:55:07,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707686 2024-11-22T23:55:07,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707686 2024-11-22T23:55:07,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707686) roll requested 2024-11-22T23:55:07,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707693 2024-11-22T23:55:07,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707693 2024-11-22T23:55:07,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707693) roll requested 2024-11-22T23:55:07,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707700 2024-11-22T23:55:07,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707700 2024-11-22T23:55:07,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707700) roll requested 2024-11-22T23:55:07,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707724 2024-11-22T23:55:07,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707724 2024-11-22T23:55:07,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707724) roll requested 2024-11-22T23:55:07,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707734 2024-11-22T23:55:07,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707734 2024-11-22T23:55:07,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707734) roll requested 2024-11-22T23:55:07,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707741 2024-11-22T23:55:07,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707741 2024-11-22T23:55:07,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707741) roll requested 2024-11-22T23:55:07,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707749 2024-11-22T23:55:07,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707749 2024-11-22T23:55:07,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707749) roll requested 2024-11-22T23:55:07,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707756 2024-11-22T23:55:07,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707756 2024-11-22T23:55:07,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707756) roll requested 2024-11-22T23:55:07,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707764 2024-11-22T23:55:07,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707764 2024-11-22T23:55:07,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707764) roll requested 2024-11-22T23:55:07,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707771 2024-11-22T23:55:07,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707771 2024-11-22T23:55:07,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707771) roll requested 2024-11-22T23:55:07,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707778 2024-11-22T23:55:07,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707778 2024-11-22T23:55:07,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707778) roll requested 2024-11-22T23:55:07,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707785 2024-11-22T23:55:07,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707785 2024-11-22T23:55:07,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707785) roll requested 2024-11-22T23:55:07,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707792 2024-11-22T23:55:07,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707792 2024-11-22T23:55:07,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707792) roll requested 2024-11-22T23:55:07,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707799 2024-11-22T23:55:07,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707799 2024-11-22T23:55:07,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707799) roll requested 2024-11-22T23:55:07,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707807 2024-11-22T23:55:07,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707807 2024-11-22T23:55:07,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707807) roll requested 2024-11-22T23:55:07,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707814 2024-11-22T23:55:07,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707814 2024-11-22T23:55:07,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707814) roll requested 2024-11-22T23:55:07,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707821 2024-11-22T23:55:07,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707821 2024-11-22T23:55:07,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707821) roll requested 2024-11-22T23:55:07,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707828 2024-11-22T23:55:07,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707828 2024-11-22T23:55:07,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707828) roll requested 2024-11-22T23:55:07,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707835 2024-11-22T23:55:07,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707835 2024-11-22T23:55:07,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707835) roll requested 2024-11-22T23:55:07,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707843 2024-11-22T23:55:07,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707843 2024-11-22T23:55:07,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707843) roll requested 2024-11-22T23:55:07,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707850 2024-11-22T23:55:07,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707850 2024-11-22T23:55:07,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707850) roll requested 2024-11-22T23:55:07,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707857 2024-11-22T23:55:07,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707857 2024-11-22T23:55:07,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707857) roll requested 2024-11-22T23:55:07,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707865 2024-11-22T23:55:07,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707865 2024-11-22T23:55:07,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707865) roll requested 2024-11-22T23:55:07,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707873 2024-11-22T23:55:07,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707873 2024-11-22T23:55:07,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707873) roll requested 2024-11-22T23:55:07,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707880 2024-11-22T23:55:07,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707880 2024-11-22T23:55:07,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707880) roll requested 2024-11-22T23:55:07,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707887 2024-11-22T23:55:07,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707887 2024-11-22T23:55:07,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707887) roll requested 2024-11-22T23:55:07,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707895 2024-11-22T23:55:07,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707895 2024-11-22T23:55:07,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707895) roll requested 2024-11-22T23:55:07,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707902 2024-11-22T23:55:07,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707902 2024-11-22T23:55:07,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707902) roll requested 2024-11-22T23:55:07,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707909 2024-11-22T23:55:07,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707909 2024-11-22T23:55:07,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707909) roll requested 2024-11-22T23:55:07,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707917 2024-11-22T23:55:07,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707917 2024-11-22T23:55:07,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707917) roll requested 2024-11-22T23:55:07,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707924 2024-11-22T23:55:07,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707924 2024-11-22T23:55:07,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707924) roll requested 2024-11-22T23:55:07,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707931 2024-11-22T23:55:07,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707931 2024-11-22T23:55:07,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707931) roll requested 2024-11-22T23:55:07,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707938 2024-11-22T23:55:07,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707938 2024-11-22T23:55:07,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707938) roll requested 2024-11-22T23:55:07,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707945 2024-11-22T23:55:07,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707945 2024-11-22T23:55:07,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707945) roll requested 2024-11-22T23:55:07,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707953 2024-11-22T23:55:07,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707953 2024-11-22T23:55:07,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707953) roll requested 2024-11-22T23:55:07,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707961 2024-11-22T23:55:07,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707961 2024-11-22T23:55:07,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707961) roll requested 2024-11-22T23:55:07,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707968 2024-11-22T23:55:07,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707968 2024-11-22T23:55:07,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,975 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707968) roll requested 2024-11-22T23:55:07,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707975 2024-11-22T23:55:07,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707975 2024-11-22T23:55:07,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707975) roll requested 2024-11-22T23:55:07,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707982 2024-11-22T23:55:07,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707982 2024-11-22T23:55:07,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707982) roll requested 2024-11-22T23:55:07,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707989 2024-11-22T23:55:07,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:07,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707989 2024-11-22T23:55:07,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:07,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:07,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707989) roll requested 2024-11-22T23:55:07,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319707996 2024-11-22T23:55:08,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707996 2024-11-22T23:55:08,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319707996) roll requested 2024-11-22T23:55:08,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708002 2024-11-22T23:55:08,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319707996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708002 2024-11-22T23:55:08,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708002) roll requested 2024-11-22T23:55:08,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708009 2024-11-22T23:55:08,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708009 2024-11-22T23:55:08,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708009) roll requested 2024-11-22T23:55:08,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708016 2024-11-22T23:55:08,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708016 2024-11-22T23:55:08,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708016) roll requested 2024-11-22T23:55:08,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708023 2024-11-22T23:55:08,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708023 2024-11-22T23:55:08,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708023) roll requested 2024-11-22T23:55:08,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708030 2024-11-22T23:55:08,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708030 2024-11-22T23:55:08,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708030) roll requested 2024-11-22T23:55:08,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708038 2024-11-22T23:55:08,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708038 2024-11-22T23:55:08,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708038) roll requested 2024-11-22T23:55:08,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708045 2024-11-22T23:55:08,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708045 2024-11-22T23:55:08,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708045) roll requested 2024-11-22T23:55:08,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708052 2024-11-22T23:55:08,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708052 2024-11-22T23:55:08,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708052) roll requested 2024-11-22T23:55:08,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708059 2024-11-22T23:55:08,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708059 2024-11-22T23:55:08,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708059) roll requested 2024-11-22T23:55:08,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708066 2024-11-22T23:55:08,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708066 2024-11-22T23:55:08,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708066) roll requested 2024-11-22T23:55:08,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708073 2024-11-22T23:55:08,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708073 2024-11-22T23:55:08,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708073) roll requested 2024-11-22T23:55:08,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708080 2024-11-22T23:55:08,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708080 2024-11-22T23:55:08,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708080) roll requested 2024-11-22T23:55:08,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708086 2024-11-22T23:55:08,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708086 2024-11-22T23:55:08,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708086) roll requested 2024-11-22T23:55:08,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708093 2024-11-22T23:55:08,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708093 2024-11-22T23:55:08,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708093) roll requested 2024-11-22T23:55:08,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708099 2024-11-22T23:55:08,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708099 2024-11-22T23:55:08,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708099) roll requested 2024-11-22T23:55:08,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708106 2024-11-22T23:55:08,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708106 2024-11-22T23:55:08,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708106) roll requested 2024-11-22T23:55:08,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708113 2024-11-22T23:55:08,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708113 2024-11-22T23:55:08,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708113) roll requested 2024-11-22T23:55:08,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708119 2024-11-22T23:55:08,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708119 2024-11-22T23:55:08,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708119) roll requested 2024-11-22T23:55:08,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708126 2024-11-22T23:55:08,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708126 2024-11-22T23:55:08,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708126) roll requested 2024-11-22T23:55:08,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708133 2024-11-22T23:55:08,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708133 2024-11-22T23:55:08,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708133) roll requested 2024-11-22T23:55:08,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708140 2024-11-22T23:55:08,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708140 2024-11-22T23:55:08,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708140) roll requested 2024-11-22T23:55:08,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708147 2024-11-22T23:55:08,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708147 2024-11-22T23:55:08,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708147) roll requested 2024-11-22T23:55:08,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708154 2024-11-22T23:55:08,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708154 2024-11-22T23:55:08,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708154) roll requested 2024-11-22T23:55:08,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708161 2024-11-22T23:55:08,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708161 2024-11-22T23:55:08,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708161) roll requested 2024-11-22T23:55:08,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708168 2024-11-22T23:55:08,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708168 2024-11-22T23:55:08,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,174 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708168) roll requested 2024-11-22T23:55:08,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708175 2024-11-22T23:55:08,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708175 2024-11-22T23:55:08,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708175) roll requested 2024-11-22T23:55:08,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708182 2024-11-22T23:55:08,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708182 2024-11-22T23:55:08,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708182) roll requested 2024-11-22T23:55:08,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708189 2024-11-22T23:55:08,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708189 2024-11-22T23:55:08,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708189) roll requested 2024-11-22T23:55:08,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708195 2024-11-22T23:55:08,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708195 2024-11-22T23:55:08,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708195) roll requested 2024-11-22T23:55:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708202 2024-11-22T23:55:08,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708202 2024-11-22T23:55:08,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708202) roll requested 2024-11-22T23:55:08,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708209 2024-11-22T23:55:08,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708209 2024-11-22T23:55:08,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708209) roll requested 2024-11-22T23:55:08,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708215 2024-11-22T23:55:08,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708215 2024-11-22T23:55:08,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708215) roll requested 2024-11-22T23:55:08,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708222 2024-11-22T23:55:08,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708222 2024-11-22T23:55:08,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708222) roll requested 2024-11-22T23:55:08,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708229 2024-11-22T23:55:08,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708229 2024-11-22T23:55:08,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708229) roll requested 2024-11-22T23:55:08,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708235 2024-11-22T23:55:08,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708235 2024-11-22T23:55:08,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708235) roll requested 2024-11-22T23:55:08,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708242 2024-11-22T23:55:08,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708242 2024-11-22T23:55:08,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708242) roll requested 2024-11-22T23:55:08,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708259 2024-11-22T23:55:08,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708259 2024-11-22T23:55:08,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708259) roll requested 2024-11-22T23:55:08,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708266 2024-11-22T23:55:08,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708266 2024-11-22T23:55:08,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708266) roll requested 2024-11-22T23:55:08,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708279 2024-11-22T23:55:08,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708279 2024-11-22T23:55:08,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708279) roll requested 2024-11-22T23:55:08,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708286 2024-11-22T23:55:08,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708286 2024-11-22T23:55:08,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708286) roll requested 2024-11-22T23:55:08,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708293 2024-11-22T23:55:08,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708293 2024-11-22T23:55:08,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708293) roll requested 2024-11-22T23:55:08,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708300 2024-11-22T23:55:08,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708300 2024-11-22T23:55:08,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708300) roll requested 2024-11-22T23:55:08,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708307 2024-11-22T23:55:08,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708307 2024-11-22T23:55:08,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708307) roll requested 2024-11-22T23:55:08,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708314 2024-11-22T23:55:08,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708314 2024-11-22T23:55:08,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708314) roll requested 2024-11-22T23:55:08,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708321 2024-11-22T23:55:08,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708321 2024-11-22T23:55:08,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708321) roll requested 2024-11-22T23:55:08,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708328 2024-11-22T23:55:08,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708328 2024-11-22T23:55:08,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708328) roll requested 2024-11-22T23:55:08,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708334 2024-11-22T23:55:08,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708334 2024-11-22T23:55:08,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708334) roll requested 2024-11-22T23:55:08,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708345 2024-11-22T23:55:08,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708345 2024-11-22T23:55:08,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708345) roll requested 2024-11-22T23:55:08,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708353 2024-11-22T23:55:08,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708353 2024-11-22T23:55:08,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708353) roll requested 2024-11-22T23:55:08,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708361 2024-11-22T23:55:08,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708361 2024-11-22T23:55:08,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708361) roll requested 2024-11-22T23:55:08,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708368 2024-11-22T23:55:08,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708368 2024-11-22T23:55:08,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708368) roll requested 2024-11-22T23:55:08,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708383 2024-11-22T23:55:08,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708383 2024-11-22T23:55:08,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708383) roll requested 2024-11-22T23:55:08,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708390 2024-11-22T23:55:08,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708390 2024-11-22T23:55:08,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708390) roll requested 2024-11-22T23:55:08,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708401 2024-11-22T23:55:08,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708401 2024-11-22T23:55:08,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708401) roll requested 2024-11-22T23:55:08,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708409 2024-11-22T23:55:08,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708409 2024-11-22T23:55:08,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708409) roll requested 2024-11-22T23:55:08,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708415 2024-11-22T23:55:08,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708415 2024-11-22T23:55:08,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708415) roll requested 2024-11-22T23:55:08,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708422 2024-11-22T23:55:08,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708422 2024-11-22T23:55:08,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708422) roll requested 2024-11-22T23:55:08,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708430 2024-11-22T23:55:08,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708430 2024-11-22T23:55:08,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708430) roll requested 2024-11-22T23:55:08,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708440 2024-11-22T23:55:08,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708440 2024-11-22T23:55:08,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708440) roll requested 2024-11-22T23:55:08,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708447 2024-11-22T23:55:08,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708447 2024-11-22T23:55:08,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708447) roll requested 2024-11-22T23:55:08,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708454 2024-11-22T23:55:08,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708454 2024-11-22T23:55:08,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708454) roll requested 2024-11-22T23:55:08,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708462 2024-11-22T23:55:08,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708462 2024-11-22T23:55:08,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708462) roll requested 2024-11-22T23:55:08,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708468 2024-11-22T23:55:08,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708468 2024-11-22T23:55:08,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708468) roll requested 2024-11-22T23:55:08,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708476 2024-11-22T23:55:08,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708476 2024-11-22T23:55:08,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708476) roll requested 2024-11-22T23:55:08,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708483 2024-11-22T23:55:08,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708483 2024-11-22T23:55:08,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708483) roll requested 2024-11-22T23:55:08,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708489 2024-11-22T23:55:08,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708489 2024-11-22T23:55:08,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708489) roll requested 2024-11-22T23:55:08,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708496 2024-11-22T23:55:08,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708496 2024-11-22T23:55:08,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708496) roll requested 2024-11-22T23:55:08,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708503 2024-11-22T23:55:08,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708503 2024-11-22T23:55:08,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708503) roll requested 2024-11-22T23:55:08,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708510 2024-11-22T23:55:08,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708510 2024-11-22T23:55:08,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708510) roll requested 2024-11-22T23:55:08,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708517 2024-11-22T23:55:08,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708517 2024-11-22T23:55:08,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708517) roll requested 2024-11-22T23:55:08,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708525 2024-11-22T23:55:08,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708525 2024-11-22T23:55:08,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708525) roll requested 2024-11-22T23:55:08,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708533 2024-11-22T23:55:08,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708533 2024-11-22T23:55:08,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708533) roll requested 2024-11-22T23:55:08,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708542 2024-11-22T23:55:08,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708542 2024-11-22T23:55:08,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708542) roll requested 2024-11-22T23:55:08,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708549 2024-11-22T23:55:08,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708549 2024-11-22T23:55:08,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708549) roll requested 2024-11-22T23:55:08,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708557 2024-11-22T23:55:08,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708557 2024-11-22T23:55:08,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708557) roll requested 2024-11-22T23:55:08,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708564 2024-11-22T23:55:08,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708564 2024-11-22T23:55:08,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708564) roll requested 2024-11-22T23:55:08,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708572 2024-11-22T23:55:08,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708572 2024-11-22T23:55:08,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708572) roll requested 2024-11-22T23:55:08,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708578 2024-11-22T23:55:08,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708578 2024-11-22T23:55:08,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708578) roll requested 2024-11-22T23:55:08,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708585 2024-11-22T23:55:08,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708585 2024-11-22T23:55:08,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708585) roll requested 2024-11-22T23:55:08,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708593 2024-11-22T23:55:08,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708593 2024-11-22T23:55:08,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708593) roll requested 2024-11-22T23:55:08,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708599 2024-11-22T23:55:08,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708599 2024-11-22T23:55:08,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708599) roll requested 2024-11-22T23:55:08,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708606 2024-11-22T23:55:08,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708606 2024-11-22T23:55:08,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708606) roll requested 2024-11-22T23:55:08,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708614 2024-11-22T23:55:08,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708614 2024-11-22T23:55:08,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708614) roll requested 2024-11-22T23:55:08,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708621 2024-11-22T23:55:08,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708621 2024-11-22T23:55:08,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708621) roll requested 2024-11-22T23:55:08,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708628 2024-11-22T23:55:08,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708628 2024-11-22T23:55:08,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708628) roll requested 2024-11-22T23:55:08,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708634 2024-11-22T23:55:08,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708634 2024-11-22T23:55:08,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708634) roll requested 2024-11-22T23:55:08,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708641 2024-11-22T23:55:08,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708641 2024-11-22T23:55:08,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708641) roll requested 2024-11-22T23:55:08,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708648 2024-11-22T23:55:08,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708648 2024-11-22T23:55:08,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708648) roll requested 2024-11-22T23:55:08,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708654 2024-11-22T23:55:08,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708654 2024-11-22T23:55:08,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708654) roll requested 2024-11-22T23:55:08,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708661 2024-11-22T23:55:08,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708661 2024-11-22T23:55:08,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708661) roll requested 2024-11-22T23:55:08,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708668 2024-11-22T23:55:08,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708668 2024-11-22T23:55:08,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708668) roll requested 2024-11-22T23:55:08,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708674 2024-11-22T23:55:08,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708674 2024-11-22T23:55:08,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708674) roll requested 2024-11-22T23:55:08,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708681 2024-11-22T23:55:08,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708681 2024-11-22T23:55:08,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708681) roll requested 2024-11-22T23:55:08,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708690 2024-11-22T23:55:08,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708690 2024-11-22T23:55:08,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708690) roll requested 2024-11-22T23:55:08,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708696 2024-11-22T23:55:08,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708696 2024-11-22T23:55:08,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708696) roll requested 2024-11-22T23:55:08,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708704 2024-11-22T23:55:08,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708704 2024-11-22T23:55:08,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708704) roll requested 2024-11-22T23:55:08,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708711 2024-11-22T23:55:08,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708711 2024-11-22T23:55:08,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708711) roll requested 2024-11-22T23:55:08,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708723 2024-11-22T23:55:08,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708723 2024-11-22T23:55:08,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708723) roll requested 2024-11-22T23:55:08,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708729 2024-11-22T23:55:08,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708729 2024-11-22T23:55:08,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708729) roll requested 2024-11-22T23:55:08,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708736 2024-11-22T23:55:08,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708736 2024-11-22T23:55:08,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708736) roll requested 2024-11-22T23:55:08,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708742 2024-11-22T23:55:08,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708742 2024-11-22T23:55:08,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708742) roll requested 2024-11-22T23:55:08,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708750 2024-11-22T23:55:08,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708750 2024-11-22T23:55:08,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708750) roll requested 2024-11-22T23:55:08,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708756 2024-11-22T23:55:08,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708756 2024-11-22T23:55:08,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708756) roll requested 2024-11-22T23:55:08,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708763 2024-11-22T23:55:08,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708763 2024-11-22T23:55:08,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708763) roll requested 2024-11-22T23:55:08,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708770 2024-11-22T23:55:08,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708770 2024-11-22T23:55:08,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708770) roll requested 2024-11-22T23:55:08,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708777 2024-11-22T23:55:08,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708777 2024-11-22T23:55:08,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708777) roll requested 2024-11-22T23:55:08,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708783 2024-11-22T23:55:08,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708783 2024-11-22T23:55:08,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708783) roll requested 2024-11-22T23:55:08,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708791 2024-11-22T23:55:08,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708791 2024-11-22T23:55:08,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708791) roll requested 2024-11-22T23:55:08,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708797 2024-11-22T23:55:08,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708797 2024-11-22T23:55:08,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708797) roll requested 2024-11-22T23:55:08,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708804 2024-11-22T23:55:08,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708804 2024-11-22T23:55:08,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708804) roll requested 2024-11-22T23:55:08,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708811 2024-11-22T23:55:08,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708811 2024-11-22T23:55:08,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708811) roll requested 2024-11-22T23:55:08,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708818 2024-11-22T23:55:08,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708818 2024-11-22T23:55:08,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708818) roll requested 2024-11-22T23:55:08,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708825 2024-11-22T23:55:08,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708825 2024-11-22T23:55:08,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708825) roll requested 2024-11-22T23:55:08,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708832 2024-11-22T23:55:08,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708832 2024-11-22T23:55:08,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708832) roll requested 2024-11-22T23:55:08,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708838 2024-11-22T23:55:08,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708838 2024-11-22T23:55:08,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708838) roll requested 2024-11-22T23:55:08,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708844 2024-11-22T23:55:08,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708844 2024-11-22T23:55:08,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708844) roll requested 2024-11-22T23:55:08,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708852 2024-11-22T23:55:08,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708852 2024-11-22T23:55:08,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708852) roll requested 2024-11-22T23:55:08,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708858 2024-11-22T23:55:08,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708858 2024-11-22T23:55:08,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708858) roll requested 2024-11-22T23:55:08,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708865 2024-11-22T23:55:08,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708865 2024-11-22T23:55:08,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708865) roll requested 2024-11-22T23:55:08,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708872 2024-11-22T23:55:08,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708872 2024-11-22T23:55:08,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708872) roll requested 2024-11-22T23:55:08,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708879 2024-11-22T23:55:08,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708879 2024-11-22T23:55:08,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708879) roll requested 2024-11-22T23:55:08,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708886 2024-11-22T23:55:08,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708886 2024-11-22T23:55:08,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708886) roll requested 2024-11-22T23:55:08,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708892 2024-11-22T23:55:08,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708892 2024-11-22T23:55:08,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708892) roll requested 2024-11-22T23:55:08,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708899 2024-11-22T23:55:08,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708899 2024-11-22T23:55:08,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708899) roll requested 2024-11-22T23:55:08,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708906 2024-11-22T23:55:08,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708906 2024-11-22T23:55:08,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708906) roll requested 2024-11-22T23:55:08,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708914 2024-11-22T23:55:08,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708914 2024-11-22T23:55:08,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708914) roll requested 2024-11-22T23:55:08,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708921 2024-11-22T23:55:08,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708921 2024-11-22T23:55:08,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708921) roll requested 2024-11-22T23:55:08,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708928 2024-11-22T23:55:08,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708928 2024-11-22T23:55:08,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708928) roll requested 2024-11-22T23:55:08,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708935 2024-11-22T23:55:08,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708935 2024-11-22T23:55:08,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708935) roll requested 2024-11-22T23:55:08,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708942 2024-11-22T23:55:08,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708942 2024-11-22T23:55:08,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708942) roll requested 2024-11-22T23:55:08,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708949 2024-11-22T23:55:08,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708949 2024-11-22T23:55:08,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708949) roll requested 2024-11-22T23:55:08,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708955 2024-11-22T23:55:08,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708955 2024-11-22T23:55:08,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708955) roll requested 2024-11-22T23:55:08,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708962 2024-11-22T23:55:08,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708962 2024-11-22T23:55:08,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708962) roll requested 2024-11-22T23:55:08,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708969 2024-11-22T23:55:08,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708969 2024-11-22T23:55:08,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708969) roll requested 2024-11-22T23:55:08,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708976 2024-11-22T23:55:08,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708976 2024-11-22T23:55:08,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708976) roll requested 2024-11-22T23:55:08,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708983 2024-11-22T23:55:08,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708983 2024-11-22T23:55:08,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708983) roll requested 2024-11-22T23:55:08,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708990 2024-11-22T23:55:08,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:08,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708990 2024-11-22T23:55:08,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:08,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:08,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708990) roll requested 2024-11-22T23:55:08,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319708997 2024-11-22T23:55:09,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708997 2024-11-22T23:55:09,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319708997) roll requested 2024-11-22T23:55:09,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709004 2024-11-22T23:55:09,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319708997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709004 2024-11-22T23:55:09,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709004) roll requested 2024-11-22T23:55:09,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709011 2024-11-22T23:55:09,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709011 2024-11-22T23:55:09,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709011) roll requested 2024-11-22T23:55:09,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709018 2024-11-22T23:55:09,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709018 2024-11-22T23:55:09,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709018) roll requested 2024-11-22T23:55:09,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709027 2024-11-22T23:55:09,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709027 2024-11-22T23:55:09,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709027) roll requested 2024-11-22T23:55:09,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709034 2024-11-22T23:55:09,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709034 2024-11-22T23:55:09,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709034) roll requested 2024-11-22T23:55:09,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709041 2024-11-22T23:55:09,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709041 2024-11-22T23:55:09,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709041) roll requested 2024-11-22T23:55:09,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709048 2024-11-22T23:55:09,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709048 2024-11-22T23:55:09,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709048) roll requested 2024-11-22T23:55:09,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709055 2024-11-22T23:55:09,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709055 2024-11-22T23:55:09,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709055) roll requested 2024-11-22T23:55:09,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709062 2024-11-22T23:55:09,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709062 2024-11-22T23:55:09,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709062) roll requested 2024-11-22T23:55:09,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709069 2024-11-22T23:55:09,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709069 2024-11-22T23:55:09,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709069) roll requested 2024-11-22T23:55:09,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709076 2024-11-22T23:55:09,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709076 2024-11-22T23:55:09,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709076) roll requested 2024-11-22T23:55:09,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709082 2024-11-22T23:55:09,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709082 2024-11-22T23:55:09,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709082) roll requested 2024-11-22T23:55:09,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709089 2024-11-22T23:55:09,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709089 2024-11-22T23:55:09,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709089) roll requested 2024-11-22T23:55:09,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709096 2024-11-22T23:55:09,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709096 2024-11-22T23:55:09,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709096) roll requested 2024-11-22T23:55:09,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709103 2024-11-22T23:55:09,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709103 2024-11-22T23:55:09,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709103) roll requested 2024-11-22T23:55:09,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709110 2024-11-22T23:55:09,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709110 2024-11-22T23:55:09,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709110) roll requested 2024-11-22T23:55:09,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709118 2024-11-22T23:55:09,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709118 2024-11-22T23:55:09,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709118) roll requested 2024-11-22T23:55:09,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709124 2024-11-22T23:55:09,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709124 2024-11-22T23:55:09,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709124) roll requested 2024-11-22T23:55:09,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709131 2024-11-22T23:55:09,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709131 2024-11-22T23:55:09,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709131) roll requested 2024-11-22T23:55:09,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709137 2024-11-22T23:55:09,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709137 2024-11-22T23:55:09,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709137) roll requested 2024-11-22T23:55:09,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709143 2024-11-22T23:55:09,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709143 2024-11-22T23:55:09,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709143) roll requested 2024-11-22T23:55:09,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709150 2024-11-22T23:55:09,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709150 2024-11-22T23:55:09,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709150) roll requested 2024-11-22T23:55:09,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709157 2024-11-22T23:55:09,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709157 2024-11-22T23:55:09,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709157) roll requested 2024-11-22T23:55:09,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709164 2024-11-22T23:55:09,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709164 2024-11-22T23:55:09,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709164) roll requested 2024-11-22T23:55:09,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709170 2024-11-22T23:55:09,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709170 2024-11-22T23:55:09,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709170) roll requested 2024-11-22T23:55:09,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709177 2024-11-22T23:55:09,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709177 2024-11-22T23:55:09,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709177) roll requested 2024-11-22T23:55:09,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709184 2024-11-22T23:55:09,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709184 2024-11-22T23:55:09,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709184) roll requested 2024-11-22T23:55:09,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709190 2024-11-22T23:55:09,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709190 2024-11-22T23:55:09,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709190) roll requested 2024-11-22T23:55:09,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709197 2024-11-22T23:55:09,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709197 2024-11-22T23:55:09,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709197) roll requested 2024-11-22T23:55:09,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709204 2024-11-22T23:55:09,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709204 2024-11-22T23:55:09,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709204) roll requested 2024-11-22T23:55:09,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709211 2024-11-22T23:55:09,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709211 2024-11-22T23:55:09,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709211) roll requested 2024-11-22T23:55:09,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709218 2024-11-22T23:55:09,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709218 2024-11-22T23:55:09,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709218) roll requested 2024-11-22T23:55:09,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709225 2024-11-22T23:55:09,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709225 2024-11-22T23:55:09,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709225) roll requested 2024-11-22T23:55:09,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709232 2024-11-22T23:55:09,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709232 2024-11-22T23:55:09,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709232) roll requested 2024-11-22T23:55:09,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709239 2024-11-22T23:55:09,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709239 2024-11-22T23:55:09,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709239) roll requested 2024-11-22T23:55:09,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709246 2024-11-22T23:55:09,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709246 2024-11-22T23:55:09,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709246) roll requested 2024-11-22T23:55:09,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709253 2024-11-22T23:55:09,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709253 2024-11-22T23:55:09,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709253) roll requested 2024-11-22T23:55:09,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709259 2024-11-22T23:55:09,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709259 2024-11-22T23:55:09,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709259) roll requested 2024-11-22T23:55:09,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709266 2024-11-22T23:55:09,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709266 2024-11-22T23:55:09,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709266) roll requested 2024-11-22T23:55:09,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709273 2024-11-22T23:55:09,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709273 2024-11-22T23:55:09,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709273) roll requested 2024-11-22T23:55:09,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709281 2024-11-22T23:55:09,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709281 2024-11-22T23:55:09,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709281) roll requested 2024-11-22T23:55:09,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709288 2024-11-22T23:55:09,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709288 2024-11-22T23:55:09,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709288) roll requested 2024-11-22T23:55:09,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709295 2024-11-22T23:55:09,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709295 2024-11-22T23:55:09,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709295) roll requested 2024-11-22T23:55:09,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709303 2024-11-22T23:55:09,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709303 2024-11-22T23:55:09,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709303) roll requested 2024-11-22T23:55:09,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709310 2024-11-22T23:55:09,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709310 2024-11-22T23:55:09,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709310) roll requested 2024-11-22T23:55:09,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709317 2024-11-22T23:55:09,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709317 2024-11-22T23:55:09,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709317) roll requested 2024-11-22T23:55:09,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709324 2024-11-22T23:55:09,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709324 2024-11-22T23:55:09,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709324) roll requested 2024-11-22T23:55:09,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709342 2024-11-22T23:55:09,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709342 2024-11-22T23:55:09,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709342) roll requested 2024-11-22T23:55:09,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709350 2024-11-22T23:55:09,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709350 2024-11-22T23:55:09,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709350) roll requested 2024-11-22T23:55:09,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709357 2024-11-22T23:55:09,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709357 2024-11-22T23:55:09,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709357) roll requested 2024-11-22T23:55:09,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709364 2024-11-22T23:55:09,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709364 2024-11-22T23:55:09,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709364) roll requested 2024-11-22T23:55:09,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709371 2024-11-22T23:55:09,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709371 2024-11-22T23:55:09,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709371) roll requested 2024-11-22T23:55:09,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709378 2024-11-22T23:55:09,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709378 2024-11-22T23:55:09,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709378) roll requested 2024-11-22T23:55:09,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709384 2024-11-22T23:55:09,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709384 2024-11-22T23:55:09,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709384) roll requested 2024-11-22T23:55:09,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709391 2024-11-22T23:55:09,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709391 2024-11-22T23:55:09,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709391) roll requested 2024-11-22T23:55:09,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709398 2024-11-22T23:55:09,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709398 2024-11-22T23:55:09,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709398) roll requested 2024-11-22T23:55:09,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709405 2024-11-22T23:55:09,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709405 2024-11-22T23:55:09,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709405) roll requested 2024-11-22T23:55:09,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709412 2024-11-22T23:55:09,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709412 2024-11-22T23:55:09,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709412) roll requested 2024-11-22T23:55:09,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709420 2024-11-22T23:55:09,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709420 2024-11-22T23:55:09,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709420) roll requested 2024-11-22T23:55:09,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709427 2024-11-22T23:55:09,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709427 2024-11-22T23:55:09,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709427) roll requested 2024-11-22T23:55:09,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709434 2024-11-22T23:55:09,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709434 2024-11-22T23:55:09,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709434) roll requested 2024-11-22T23:55:09,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709440 2024-11-22T23:55:09,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709440 2024-11-22T23:55:09,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709440) roll requested 2024-11-22T23:55:09,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709447 2024-11-22T23:55:09,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709447 2024-11-22T23:55:09,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709447) roll requested 2024-11-22T23:55:09,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709454 2024-11-22T23:55:09,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709454 2024-11-22T23:55:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709447 is not closed yet, will try archiving it next time 2024-11-22T23:55:09,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709454) roll requested 2024-11-22T23:55:09,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709461 2024-11-22T23:55:09,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709461 2024-11-22T23:55:09,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709461) roll requested 2024-11-22T23:55:09,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709467 2024-11-22T23:55:09,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709467 2024-11-22T23:55:09,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,474 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709467) roll requested 2024-11-22T23:55:09,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709474 2024-11-22T23:55:09,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709474 2024-11-22T23:55:09,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709474) roll requested 2024-11-22T23:55:09,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709481 2024-11-22T23:55:09,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709481 2024-11-22T23:55:09,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709481) roll requested 2024-11-22T23:55:09,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709487 2024-11-22T23:55:09,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709487 2024-11-22T23:55:09,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709487) roll requested 2024-11-22T23:55:09,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709494 2024-11-22T23:55:09,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709494 2024-11-22T23:55:09,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709494) roll requested 2024-11-22T23:55:09,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709501 2024-11-22T23:55:09,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709501 2024-11-22T23:55:09,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709501) roll requested 2024-11-22T23:55:09,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709508 2024-11-22T23:55:09,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709508 2024-11-22T23:55:09,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709508) roll requested 2024-11-22T23:55:09,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709515 2024-11-22T23:55:09,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709515 2024-11-22T23:55:09,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709515) roll requested 2024-11-22T23:55:09,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709521 2024-11-22T23:55:09,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709521 2024-11-22T23:55:09,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709521) roll requested 2024-11-22T23:55:09,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709528 2024-11-22T23:55:09,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709528 2024-11-22T23:55:09,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709528) roll requested 2024-11-22T23:55:09,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709535 2024-11-22T23:55:09,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709535 2024-11-22T23:55:09,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709535) roll requested 2024-11-22T23:55:09,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709542 2024-11-22T23:55:09,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709542 2024-11-22T23:55:09,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709542) roll requested 2024-11-22T23:55:09,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709549 2024-11-22T23:55:09,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709549 2024-11-22T23:55:09,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709549) roll requested 2024-11-22T23:55:09,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709557 2024-11-22T23:55:09,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709557 2024-11-22T23:55:09,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709557) roll requested 2024-11-22T23:55:09,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709564 2024-11-22T23:55:09,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709564 2024-11-22T23:55:09,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709564) roll requested 2024-11-22T23:55:09,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709571 2024-11-22T23:55:09,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709571 2024-11-22T23:55:09,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709571) roll requested 2024-11-22T23:55:09,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709578 2024-11-22T23:55:09,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709578 2024-11-22T23:55:09,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709578) roll requested 2024-11-22T23:55:09,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709585 2024-11-22T23:55:09,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709585 2024-11-22T23:55:09,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709585) roll requested 2024-11-22T23:55:09,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709591 2024-11-22T23:55:09,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709591 2024-11-22T23:55:09,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709591) roll requested 2024-11-22T23:55:09,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709598 2024-11-22T23:55:09,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709598 2024-11-22T23:55:09,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709598) roll requested 2024-11-22T23:55:09,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709605 2024-11-22T23:55:09,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709605 2024-11-22T23:55:09,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709605) roll requested 2024-11-22T23:55:09,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709612 2024-11-22T23:55:09,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709612 2024-11-22T23:55:09,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709612) roll requested 2024-11-22T23:55:09,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709619 2024-11-22T23:55:09,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709619 2024-11-22T23:55:09,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709619) roll requested 2024-11-22T23:55:09,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709626 2024-11-22T23:55:09,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709626 2024-11-22T23:55:09,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709626) roll requested 2024-11-22T23:55:09,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709632 2024-11-22T23:55:09,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709632 2024-11-22T23:55:09,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709632) roll requested 2024-11-22T23:55:09,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709642 2024-11-22T23:55:09,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709642 2024-11-22T23:55:09,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709642) roll requested 2024-11-22T23:55:09,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709649 2024-11-22T23:55:09,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709649 2024-11-22T23:55:09,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709649) roll requested 2024-11-22T23:55:09,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709656 2024-11-22T23:55:09,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709656 2024-11-22T23:55:09,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709656) roll requested 2024-11-22T23:55:09,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709663 2024-11-22T23:55:09,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709663 2024-11-22T23:55:09,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709663) roll requested 2024-11-22T23:55:09,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709670 2024-11-22T23:55:09,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709670 2024-11-22T23:55:09,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709670) roll requested 2024-11-22T23:55:09,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709677 2024-11-22T23:55:09,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709677 2024-11-22T23:55:09,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709677) roll requested 2024-11-22T23:55:09,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709684 2024-11-22T23:55:09,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709684 2024-11-22T23:55:09,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709684) roll requested 2024-11-22T23:55:09,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709690 2024-11-22T23:55:09,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709690 2024-11-22T23:55:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709690) roll requested 2024-11-22T23:55:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709697 2024-11-22T23:55:09,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709697 2024-11-22T23:55:09,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709697) roll requested 2024-11-22T23:55:09,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709704 2024-11-22T23:55:09,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709704 2024-11-22T23:55:09,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709704) roll requested 2024-11-22T23:55:09,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709711 2024-11-22T23:55:09,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709711 2024-11-22T23:55:09,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709711) roll requested 2024-11-22T23:55:09,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709718 2024-11-22T23:55:09,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709718 2024-11-22T23:55:09,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709718) roll requested 2024-11-22T23:55:09,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709726 2024-11-22T23:55:09,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709726 2024-11-22T23:55:09,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709726) roll requested 2024-11-22T23:55:09,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709733 2024-11-22T23:55:09,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709733 2024-11-22T23:55:09,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709733) roll requested 2024-11-22T23:55:09,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709740 2024-11-22T23:55:09,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709740 2024-11-22T23:55:09,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709740) roll requested 2024-11-22T23:55:09,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709746 2024-11-22T23:55:09,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709746 2024-11-22T23:55:09,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709746) roll requested 2024-11-22T23:55:09,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709753 2024-11-22T23:55:09,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709753 2024-11-22T23:55:09,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709753) roll requested 2024-11-22T23:55:09,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709759 2024-11-22T23:55:09,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709759 2024-11-22T23:55:09,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709759) roll requested 2024-11-22T23:55:09,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709766 2024-11-22T23:55:09,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709766 2024-11-22T23:55:09,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709766) roll requested 2024-11-22T23:55:09,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709773 2024-11-22T23:55:09,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709773 2024-11-22T23:55:09,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709773) roll requested 2024-11-22T23:55:09,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709780 2024-11-22T23:55:09,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709780 2024-11-22T23:55:09,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709780) roll requested 2024-11-22T23:55:09,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709787 2024-11-22T23:55:09,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709787 2024-11-22T23:55:09,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709787) roll requested 2024-11-22T23:55:09,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709794 2024-11-22T23:55:09,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709794 2024-11-22T23:55:09,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709794) roll requested 2024-11-22T23:55:09,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709801 2024-11-22T23:55:09,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709801 2024-11-22T23:55:09,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709801) roll requested 2024-11-22T23:55:09,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709807 2024-11-22T23:55:09,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709807 2024-11-22T23:55:09,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709807) roll requested 2024-11-22T23:55:09,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709814 2024-11-22T23:55:09,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709814 2024-11-22T23:55:09,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709814) roll requested 2024-11-22T23:55:09,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709821 2024-11-22T23:55:09,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709821 2024-11-22T23:55:09,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709821) roll requested 2024-11-22T23:55:09,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709828 2024-11-22T23:55:09,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709828 2024-11-22T23:55:09,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709828) roll requested 2024-11-22T23:55:09,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709835 2024-11-22T23:55:09,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709835 2024-11-22T23:55:09,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709835) roll requested 2024-11-22T23:55:09,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709846 2024-11-22T23:55:09,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709846 2024-11-22T23:55:09,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709846) roll requested 2024-11-22T23:55:09,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709853 2024-11-22T23:55:09,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709853 2024-11-22T23:55:09,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709853) roll requested 2024-11-22T23:55:09,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709860 2024-11-22T23:55:09,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709860 2024-11-22T23:55:09,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709860) roll requested 2024-11-22T23:55:09,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709867 2024-11-22T23:55:09,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709867 2024-11-22T23:55:09,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709867) roll requested 2024-11-22T23:55:09,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709874 2024-11-22T23:55:09,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709874 2024-11-22T23:55:09,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709874) roll requested 2024-11-22T23:55:09,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709880 2024-11-22T23:55:09,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709880 2024-11-22T23:55:09,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709880) roll requested 2024-11-22T23:55:09,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709887 2024-11-22T23:55:09,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709887 2024-11-22T23:55:09,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709887) roll requested 2024-11-22T23:55:09,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709894 2024-11-22T23:55:09,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709894 2024-11-22T23:55:09,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709894) roll requested 2024-11-22T23:55:09,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709900 2024-11-22T23:55:09,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709900 2024-11-22T23:55:09,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709900) roll requested 2024-11-22T23:55:09,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709907 2024-11-22T23:55:09,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709907 2024-11-22T23:55:09,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709907) roll requested 2024-11-22T23:55:09,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709914 2024-11-22T23:55:09,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709914 2024-11-22T23:55:09,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709914) roll requested 2024-11-22T23:55:09,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709921 2024-11-22T23:55:09,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709921 2024-11-22T23:55:09,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709921) roll requested 2024-11-22T23:55:09,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709929 2024-11-22T23:55:09,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709929 2024-11-22T23:55:09,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709929) roll requested 2024-11-22T23:55:09,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709935 2024-11-22T23:55:09,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709935 2024-11-22T23:55:09,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709935) roll requested 2024-11-22T23:55:09,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709942 2024-11-22T23:55:09,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709942 2024-11-22T23:55:09,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709942) roll requested 2024-11-22T23:55:09,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709949 2024-11-22T23:55:09,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709949 2024-11-22T23:55:09,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709949) roll requested 2024-11-22T23:55:09,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709956 2024-11-22T23:55:09,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709956 2024-11-22T23:55:09,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709956) roll requested 2024-11-22T23:55:09,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709963 2024-11-22T23:55:09,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709963 2024-11-22T23:55:09,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709963) roll requested 2024-11-22T23:55:09,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709969 2024-11-22T23:55:09,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709969 2024-11-22T23:55:09,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709969) roll requested 2024-11-22T23:55:09,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709976 2024-11-22T23:55:09,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709976 2024-11-22T23:55:09,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709976) roll requested 2024-11-22T23:55:09,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709983 2024-11-22T23:55:09,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709983 2024-11-22T23:55:09,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709983) roll requested 2024-11-22T23:55:09,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709990 2024-11-22T23:55:09,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:09,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709990 2024-11-22T23:55:09,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:09,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:09,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709990) roll requested 2024-11-22T23:55:09,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319709996 2024-11-22T23:55:10,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709990 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709996 2024-11-22T23:55:10,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319709996) roll requested 2024-11-22T23:55:10,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710003 2024-11-22T23:55:10,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319709996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710003 2024-11-22T23:55:10,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710003) roll requested 2024-11-22T23:55:10,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710010 2024-11-22T23:55:10,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710010 2024-11-22T23:55:10,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710010) roll requested 2024-11-22T23:55:10,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710017 2024-11-22T23:55:10,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710017 2024-11-22T23:55:10,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710017) roll requested 2024-11-22T23:55:10,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710024 2024-11-22T23:55:10,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710024 2024-11-22T23:55:10,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710024) roll requested 2024-11-22T23:55:10,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710031 2024-11-22T23:55:10,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710031 2024-11-22T23:55:10,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710031) roll requested 2024-11-22T23:55:10,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710038 2024-11-22T23:55:10,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710038 2024-11-22T23:55:10,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710038) roll requested 2024-11-22T23:55:10,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710045 2024-11-22T23:55:10,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710045 2024-11-22T23:55:10,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710045) roll requested 2024-11-22T23:55:10,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710052 2024-11-22T23:55:10,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710052 2024-11-22T23:55:10,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710052) roll requested 2024-11-22T23:55:10,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710059 2024-11-22T23:55:10,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710059 2024-11-22T23:55:10,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710059) roll requested 2024-11-22T23:55:10,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710066 2024-11-22T23:55:10,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710066 2024-11-22T23:55:10,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710066) roll requested 2024-11-22T23:55:10,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710072 2024-11-22T23:55:10,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710072 2024-11-22T23:55:10,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710072) roll requested 2024-11-22T23:55:10,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710079 2024-11-22T23:55:10,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710079 2024-11-22T23:55:10,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710079) roll requested 2024-11-22T23:55:10,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710086 2024-11-22T23:55:10,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710086 2024-11-22T23:55:10,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710086) roll requested 2024-11-22T23:55:10,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710092 2024-11-22T23:55:10,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710092 2024-11-22T23:55:10,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710092) roll requested 2024-11-22T23:55:10,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710099 2024-11-22T23:55:10,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710099 2024-11-22T23:55:10,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710099) roll requested 2024-11-22T23:55:10,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710106 2024-11-22T23:55:10,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710106 2024-11-22T23:55:10,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710106) roll requested 2024-11-22T23:55:10,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710113 2024-11-22T23:55:10,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710113 2024-11-22T23:55:10,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710113) roll requested 2024-11-22T23:55:10,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710119 2024-11-22T23:55:10,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710119 2024-11-22T23:55:10,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710119) roll requested 2024-11-22T23:55:10,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710126 2024-11-22T23:55:10,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710126 2024-11-22T23:55:10,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710126) roll requested 2024-11-22T23:55:10,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710134 2024-11-22T23:55:10,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710134 2024-11-22T23:55:10,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710134) roll requested 2024-11-22T23:55:10,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710141 2024-11-22T23:55:10,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710141 2024-11-22T23:55:10,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710141) roll requested 2024-11-22T23:55:10,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710148 2024-11-22T23:55:10,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710148 2024-11-22T23:55:10,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710148) roll requested 2024-11-22T23:55:10,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710154 2024-11-22T23:55:10,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710154 2024-11-22T23:55:10,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710154) roll requested 2024-11-22T23:55:10,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710161 2024-11-22T23:55:10,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710161 2024-11-22T23:55:10,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710161) roll requested 2024-11-22T23:55:10,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710168 2024-11-22T23:55:10,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710168 2024-11-22T23:55:10,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710168) roll requested 2024-11-22T23:55:10,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710175 2024-11-22T23:55:10,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710175 2024-11-22T23:55:10,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710175) roll requested 2024-11-22T23:55:10,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710182 2024-11-22T23:55:10,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710182 2024-11-22T23:55:10,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710182) roll requested 2024-11-22T23:55:10,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710189 2024-11-22T23:55:10,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710189 2024-11-22T23:55:10,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710189) roll requested 2024-11-22T23:55:10,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710196 2024-11-22T23:55:10,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710196 2024-11-22T23:55:10,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710196) roll requested 2024-11-22T23:55:10,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710202 2024-11-22T23:55:10,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710202 2024-11-22T23:55:10,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710202) roll requested 2024-11-22T23:55:10,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710209 2024-11-22T23:55:10,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710209 2024-11-22T23:55:10,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710209) roll requested 2024-11-22T23:55:10,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710216 2024-11-22T23:55:10,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710216 2024-11-22T23:55:10,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710216) roll requested 2024-11-22T23:55:10,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710224 2024-11-22T23:55:10,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710224 2024-11-22T23:55:10,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710224) roll requested 2024-11-22T23:55:10,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710231 2024-11-22T23:55:10,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710231 2024-11-22T23:55:10,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710231) roll requested 2024-11-22T23:55:10,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710238 2024-11-22T23:55:10,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710238 2024-11-22T23:55:10,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710238) roll requested 2024-11-22T23:55:10,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710245 2024-11-22T23:55:10,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710245 2024-11-22T23:55:10,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710245) roll requested 2024-11-22T23:55:10,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710253 2024-11-22T23:55:10,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710253 2024-11-22T23:55:10,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710253) roll requested 2024-11-22T23:55:10,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710259 2024-11-22T23:55:10,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710259 2024-11-22T23:55:10,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710259) roll requested 2024-11-22T23:55:10,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710267 2024-11-22T23:55:10,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710267 2024-11-22T23:55:10,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710267) roll requested 2024-11-22T23:55:10,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710274 2024-11-22T23:55:10,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710274 2024-11-22T23:55:10,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710274) roll requested 2024-11-22T23:55:10,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710280 2024-11-22T23:55:10,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710280 2024-11-22T23:55:10,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710280) roll requested 2024-11-22T23:55:10,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710287 2024-11-22T23:55:10,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710287 2024-11-22T23:55:10,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710287) roll requested 2024-11-22T23:55:10,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710294 2024-11-22T23:55:10,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710294 2024-11-22T23:55:10,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710294) roll requested 2024-11-22T23:55:10,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710301 2024-11-22T23:55:10,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710301 2024-11-22T23:55:10,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710301) roll requested 2024-11-22T23:55:10,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710308 2024-11-22T23:55:10,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710308 2024-11-22T23:55:10,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710308) roll requested 2024-11-22T23:55:10,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710314 2024-11-22T23:55:10,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710314 2024-11-22T23:55:10,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710314) roll requested 2024-11-22T23:55:10,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710321 2024-11-22T23:55:10,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710321 2024-11-22T23:55:10,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710321) roll requested 2024-11-22T23:55:10,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710328 2024-11-22T23:55:10,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710328 2024-11-22T23:55:10,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710328) roll requested 2024-11-22T23:55:10,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710335 2024-11-22T23:55:10,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710335 2024-11-22T23:55:10,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710335) roll requested 2024-11-22T23:55:10,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710342 2024-11-22T23:55:10,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710342 2024-11-22T23:55:10,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710342) roll requested 2024-11-22T23:55:10,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710349 2024-11-22T23:55:10,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710349 2024-11-22T23:55:10,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710349) roll requested 2024-11-22T23:55:10,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710356 2024-11-22T23:55:10,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710356 2024-11-22T23:55:10,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710356) roll requested 2024-11-22T23:55:10,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710363 2024-11-22T23:55:10,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710363 2024-11-22T23:55:10,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710363) roll requested 2024-11-22T23:55:10,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710370 2024-11-22T23:55:10,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710370 2024-11-22T23:55:10,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710370) roll requested 2024-11-22T23:55:10,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710377 2024-11-22T23:55:10,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710377 2024-11-22T23:55:10,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710377) roll requested 2024-11-22T23:55:10,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710384 2024-11-22T23:55:10,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710384 2024-11-22T23:55:10,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710384) roll requested 2024-11-22T23:55:10,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710390 2024-11-22T23:55:10,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710390 2024-11-22T23:55:10,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710390) roll requested 2024-11-22T23:55:10,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710397 2024-11-22T23:55:10,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710397 2024-11-22T23:55:10,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710397) roll requested 2024-11-22T23:55:10,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710404 2024-11-22T23:55:10,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710404 2024-11-22T23:55:10,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710404) roll requested 2024-11-22T23:55:10,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710411 2024-11-22T23:55:10,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710411 2024-11-22T23:55:10,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710411) roll requested 2024-11-22T23:55:10,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710418 2024-11-22T23:55:10,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710418 2024-11-22T23:55:10,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710418) roll requested 2024-11-22T23:55:10,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710425 2024-11-22T23:55:10,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710425 2024-11-22T23:55:10,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710425) roll requested 2024-11-22T23:55:10,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710432 2024-11-22T23:55:10,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710432 2024-11-22T23:55:10,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710432) roll requested 2024-11-22T23:55:10,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710439 2024-11-22T23:55:10,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710439 2024-11-22T23:55:10,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710439) roll requested 2024-11-22T23:55:10,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710445 2024-11-22T23:55:10,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710445 2024-11-22T23:55:10,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710445) roll requested 2024-11-22T23:55:10,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710452 2024-11-22T23:55:10,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710452 2024-11-22T23:55:10,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710452) roll requested 2024-11-22T23:55:10,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710459 2024-11-22T23:55:10,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710459 2024-11-22T23:55:10,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710459) roll requested 2024-11-22T23:55:10,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710466 2024-11-22T23:55:10,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710466 2024-11-22T23:55:10,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710466) roll requested 2024-11-22T23:55:10,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710473 2024-11-22T23:55:10,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710473 2024-11-22T23:55:10,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710473) roll requested 2024-11-22T23:55:10,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710480 2024-11-22T23:55:10,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710480 2024-11-22T23:55:10,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710480) roll requested 2024-11-22T23:55:10,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710486 2024-11-22T23:55:10,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710486 2024-11-22T23:55:10,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710486) roll requested 2024-11-22T23:55:10,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710493 2024-11-22T23:55:10,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710493 2024-11-22T23:55:10,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710493) roll requested 2024-11-22T23:55:10,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710500 2024-11-22T23:55:10,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710500 2024-11-22T23:55:10,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710500) roll requested 2024-11-22T23:55:10,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710507 2024-11-22T23:55:10,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710507 2024-11-22T23:55:10,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710507) roll requested 2024-11-22T23:55:10,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710514 2024-11-22T23:55:10,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710514 2024-11-22T23:55:10,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710514) roll requested 2024-11-22T23:55:10,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710522 2024-11-22T23:55:10,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710522 2024-11-22T23:55:10,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710522) roll requested 2024-11-22T23:55:10,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710531 2024-11-22T23:55:10,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710531 2024-11-22T23:55:10,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710531) roll requested 2024-11-22T23:55:10,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710539 2024-11-22T23:55:10,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710539 2024-11-22T23:55:10,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710539) roll requested 2024-11-22T23:55:10,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710547 2024-11-22T23:55:10,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710547 2024-11-22T23:55:10,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710547) roll requested 2024-11-22T23:55:10,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710554 2024-11-22T23:55:10,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710554 2024-11-22T23:55:10,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710554) roll requested 2024-11-22T23:55:10,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710561 2024-11-22T23:55:10,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710561 2024-11-22T23:55:10,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710561) roll requested 2024-11-22T23:55:10,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710568 2024-11-22T23:55:10,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710568 2024-11-22T23:55:10,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710568) roll requested 2024-11-22T23:55:10,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710575 2024-11-22T23:55:10,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710575 2024-11-22T23:55:10,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710575) roll requested 2024-11-22T23:55:10,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710581 2024-11-22T23:55:10,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710581 2024-11-22T23:55:10,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710581) roll requested 2024-11-22T23:55:10,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710588 2024-11-22T23:55:10,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710588 2024-11-22T23:55:10,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710588) roll requested 2024-11-22T23:55:10,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710595 2024-11-22T23:55:10,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710595 2024-11-22T23:55:10,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710595) roll requested 2024-11-22T23:55:10,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710602 2024-11-22T23:55:10,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710602 2024-11-22T23:55:10,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710602) roll requested 2024-11-22T23:55:10,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710609 2024-11-22T23:55:10,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710609 2024-11-22T23:55:10,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710609) roll requested 2024-11-22T23:55:10,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710616 2024-11-22T23:55:10,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710616 2024-11-22T23:55:10,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710616) roll requested 2024-11-22T23:55:10,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710623 2024-11-22T23:55:10,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710623 2024-11-22T23:55:10,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710623) roll requested 2024-11-22T23:55:10,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710631 2024-11-22T23:55:10,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710631 2024-11-22T23:55:10,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710631) roll requested 2024-11-22T23:55:10,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710638 2024-11-22T23:55:10,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710638 2024-11-22T23:55:10,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710638) roll requested 2024-11-22T23:55:10,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710652 2024-11-22T23:55:10,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710652 2024-11-22T23:55:10,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710652) roll requested 2024-11-22T23:55:10,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710660 2024-11-22T23:55:10,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710660 2024-11-22T23:55:10,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710660) roll requested 2024-11-22T23:55:10,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710667 2024-11-22T23:55:10,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710667 2024-11-22T23:55:10,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710667) roll requested 2024-11-22T23:55:10,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710674 2024-11-22T23:55:10,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710674 2024-11-22T23:55:10,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710674) roll requested 2024-11-22T23:55:10,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710681 2024-11-22T23:55:10,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710681 2024-11-22T23:55:10,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710681) roll requested 2024-11-22T23:55:10,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710688 2024-11-22T23:55:10,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710688 2024-11-22T23:55:10,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710688) roll requested 2024-11-22T23:55:10,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710694 2024-11-22T23:55:10,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710694 2024-11-22T23:55:10,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710694) roll requested 2024-11-22T23:55:10,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710701 2024-11-22T23:55:10,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710701 2024-11-22T23:55:10,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710701) roll requested 2024-11-22T23:55:10,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710708 2024-11-22T23:55:10,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710708 2024-11-22T23:55:10,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710708) roll requested 2024-11-22T23:55:10,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710715 2024-11-22T23:55:10,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710715 2024-11-22T23:55:10,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710715) roll requested 2024-11-22T23:55:10,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710722 2024-11-22T23:55:10,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710722 2024-11-22T23:55:10,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710722) roll requested 2024-11-22T23:55:10,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710729 2024-11-22T23:55:10,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710729 2024-11-22T23:55:10,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710729) roll requested 2024-11-22T23:55:10,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710737 2024-11-22T23:55:10,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710737 2024-11-22T23:55:10,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710737) roll requested 2024-11-22T23:55:10,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710744 2024-11-22T23:55:10,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710744 2024-11-22T23:55:10,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710744) roll requested 2024-11-22T23:55:10,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710751 2024-11-22T23:55:10,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710751 2024-11-22T23:55:10,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710751) roll requested 2024-11-22T23:55:10,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710758 2024-11-22T23:55:10,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710758 2024-11-22T23:55:10,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710758) roll requested 2024-11-22T23:55:10,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710765 2024-11-22T23:55:10,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710765 2024-11-22T23:55:10,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710765) roll requested 2024-11-22T23:55:10,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710772 2024-11-22T23:55:10,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710772 2024-11-22T23:55:10,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710772) roll requested 2024-11-22T23:55:10,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710779 2024-11-22T23:55:10,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710779 2024-11-22T23:55:10,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710779) roll requested 2024-11-22T23:55:10,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710786 2024-11-22T23:55:10,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710786 2024-11-22T23:55:10,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710786) roll requested 2024-11-22T23:55:10,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710793 2024-11-22T23:55:10,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710793 2024-11-22T23:55:10,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710793) roll requested 2024-11-22T23:55:10,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710799 2024-11-22T23:55:10,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710799 2024-11-22T23:55:10,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710799) roll requested 2024-11-22T23:55:10,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710806 2024-11-22T23:55:10,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710806 2024-11-22T23:55:10,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710806) roll requested 2024-11-22T23:55:10,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710833 2024-11-22T23:55:10,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710833 2024-11-22T23:55:10,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710833) roll requested 2024-11-22T23:55:10,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710851 2024-11-22T23:55:10,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710851 2024-11-22T23:55:10,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710851) roll requested 2024-11-22T23:55:10,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710862 2024-11-22T23:55:10,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710862 2024-11-22T23:55:10,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710862) roll requested 2024-11-22T23:55:10,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710869 2024-11-22T23:55:10,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710869 2024-11-22T23:55:10,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710869) roll requested 2024-11-22T23:55:10,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710876 2024-11-22T23:55:10,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710876 2024-11-22T23:55:10,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710876) roll requested 2024-11-22T23:55:10,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710890 2024-11-22T23:55:10,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710890 2024-11-22T23:55:10,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710890) roll requested 2024-11-22T23:55:10,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710899 2024-11-22T23:55:10,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710899 2024-11-22T23:55:10,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710899) roll requested 2024-11-22T23:55:10,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710906 2024-11-22T23:55:10,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710906 2024-11-22T23:55:10,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710906) roll requested 2024-11-22T23:55:10,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710914 2024-11-22T23:55:10,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710914 2024-11-22T23:55:10,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710914) roll requested 2024-11-22T23:55:10,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710920 2024-11-22T23:55:10,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710920 2024-11-22T23:55:10,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710920) roll requested 2024-11-22T23:55:10,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710928 2024-11-22T23:55:10,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710928 2024-11-22T23:55:10,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710928) roll requested 2024-11-22T23:55:10,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710934 2024-11-22T23:55:10,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710934 2024-11-22T23:55:10,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710934) roll requested 2024-11-22T23:55:10,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710942 2024-11-22T23:55:10,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710942 2024-11-22T23:55:10,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710942) roll requested 2024-11-22T23:55:10,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710949 2024-11-22T23:55:10,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710949 2024-11-22T23:55:10,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710949) roll requested 2024-11-22T23:55:10,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710956 2024-11-22T23:55:10,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710956 2024-11-22T23:55:10,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710956) roll requested 2024-11-22T23:55:10,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710963 2024-11-22T23:55:10,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710963 2024-11-22T23:55:10,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710963) roll requested 2024-11-22T23:55:10,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710970 2024-11-22T23:55:10,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710970 2024-11-22T23:55:10,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710970) roll requested 2024-11-22T23:55:10,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710977 2024-11-22T23:55:10,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710977 2024-11-22T23:55:10,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710977) roll requested 2024-11-22T23:55:10,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710984 2024-11-22T23:55:10,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710984 2024-11-22T23:55:10,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710984) roll requested 2024-11-22T23:55:10,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710991 2024-11-22T23:55:10,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:10,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710991 2024-11-22T23:55:10,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:10,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:10,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710991) roll requested 2024-11-22T23:55:10,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319710997 2024-11-22T23:55:11,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710997 2024-11-22T23:55:11,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319710997) roll requested 2024-11-22T23:55:11,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711004 2024-11-22T23:55:11,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319710997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711004 2024-11-22T23:55:11,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711004) roll requested 2024-11-22T23:55:11,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711011 2024-11-22T23:55:11,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711011 2024-11-22T23:55:11,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711011) roll requested 2024-11-22T23:55:11,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711018 2024-11-22T23:55:11,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711018 2024-11-22T23:55:11,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711018) roll requested 2024-11-22T23:55:11,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711025 2024-11-22T23:55:11,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711025 2024-11-22T23:55:11,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711025) roll requested 2024-11-22T23:55:11,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711031 2024-11-22T23:55:11,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711031 2024-11-22T23:55:11,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711031) roll requested 2024-11-22T23:55:11,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711038 2024-11-22T23:55:11,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711038 2024-11-22T23:55:11,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711038) roll requested 2024-11-22T23:55:11,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711045 2024-11-22T23:55:11,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711045 2024-11-22T23:55:11,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711045) roll requested 2024-11-22T23:55:11,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711053 2024-11-22T23:55:11,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711053 2024-11-22T23:55:11,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711053) roll requested 2024-11-22T23:55:11,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711059 2024-11-22T23:55:11,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711059 2024-11-22T23:55:11,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711059) roll requested 2024-11-22T23:55:11,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711066 2024-11-22T23:55:11,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711066 2024-11-22T23:55:11,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711066) roll requested 2024-11-22T23:55:11,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711073 2024-11-22T23:55:11,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711073 2024-11-22T23:55:11,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711073) roll requested 2024-11-22T23:55:11,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711080 2024-11-22T23:55:11,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711080 2024-11-22T23:55:11,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711080) roll requested 2024-11-22T23:55:11,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711086 2024-11-22T23:55:11,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711086 2024-11-22T23:55:11,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711086) roll requested 2024-11-22T23:55:11,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711093 2024-11-22T23:55:11,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711093 2024-11-22T23:55:11,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711093) roll requested 2024-11-22T23:55:11,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711100 2024-11-22T23:55:11,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711100 2024-11-22T23:55:11,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711100) roll requested 2024-11-22T23:55:11,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711107 2024-11-22T23:55:11,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711107 2024-11-22T23:55:11,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711107) roll requested 2024-11-22T23:55:11,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711119 2024-11-22T23:55:11,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711119 2024-11-22T23:55:11,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711119) roll requested 2024-11-22T23:55:11,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711126 2024-11-22T23:55:11,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711126 2024-11-22T23:55:11,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711126) roll requested 2024-11-22T23:55:11,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711134 2024-11-22T23:55:11,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711134 2024-11-22T23:55:11,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711134) roll requested 2024-11-22T23:55:11,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711142 2024-11-22T23:55:11,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711142 2024-11-22T23:55:11,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711142) roll requested 2024-11-22T23:55:11,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711149 2024-11-22T23:55:11,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711149 2024-11-22T23:55:11,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711149) roll requested 2024-11-22T23:55:11,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711156 2024-11-22T23:55:11,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711156 2024-11-22T23:55:11,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711156) roll requested 2024-11-22T23:55:11,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711163 2024-11-22T23:55:11,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711163 2024-11-22T23:55:11,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711163) roll requested 2024-11-22T23:55:11,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711171 2024-11-22T23:55:11,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711171 2024-11-22T23:55:11,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711171) roll requested 2024-11-22T23:55:11,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711178 2024-11-22T23:55:11,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711178 2024-11-22T23:55:11,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711178) roll requested 2024-11-22T23:55:11,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711184 2024-11-22T23:55:11,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711184 2024-11-22T23:55:11,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711184) roll requested 2024-11-22T23:55:11,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711191 2024-11-22T23:55:11,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711191 2024-11-22T23:55:11,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711191) roll requested 2024-11-22T23:55:11,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711198 2024-11-22T23:55:11,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711198 2024-11-22T23:55:11,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711198) roll requested 2024-11-22T23:55:11,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711206 2024-11-22T23:55:11,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711206 2024-11-22T23:55:11,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711206) roll requested 2024-11-22T23:55:11,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711212 2024-11-22T23:55:11,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711212 2024-11-22T23:55:11,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711212) roll requested 2024-11-22T23:55:11,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711219 2024-11-22T23:55:11,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711219 2024-11-22T23:55:11,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711219) roll requested 2024-11-22T23:55:11,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711227 2024-11-22T23:55:11,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711227 2024-11-22T23:55:11,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711227) roll requested 2024-11-22T23:55:11,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711240 2024-11-22T23:55:11,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711240 2024-11-22T23:55:11,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711240) roll requested 2024-11-22T23:55:11,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711246 2024-11-22T23:55:11,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711246 2024-11-22T23:55:11,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711246) roll requested 2024-11-22T23:55:11,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711254 2024-11-22T23:55:11,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711254 2024-11-22T23:55:11,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711254) roll requested 2024-11-22T23:55:11,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711261 2024-11-22T23:55:11,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711261 2024-11-22T23:55:11,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711261) roll requested 2024-11-22T23:55:11,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711268 2024-11-22T23:55:11,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711268 2024-11-22T23:55:11,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711268) roll requested 2024-11-22T23:55:11,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711274 2024-11-22T23:55:11,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711274 2024-11-22T23:55:11,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711274) roll requested 2024-11-22T23:55:11,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711285 2024-11-22T23:55:11,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711285 2024-11-22T23:55:11,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711285) roll requested 2024-11-22T23:55:11,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711297 2024-11-22T23:55:11,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711297 2024-11-22T23:55:11,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711297) roll requested 2024-11-22T23:55:11,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711312 2024-11-22T23:55:11,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711312 2024-11-22T23:55:11,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711312) roll requested 2024-11-22T23:55:11,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711319 2024-11-22T23:55:11,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711319 2024-11-22T23:55:11,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711319) roll requested 2024-11-22T23:55:11,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711326 2024-11-22T23:55:11,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711326 2024-11-22T23:55:11,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711326) roll requested 2024-11-22T23:55:11,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711333 2024-11-22T23:55:11,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711333 2024-11-22T23:55:11,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711333) roll requested 2024-11-22T23:55:11,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711341 2024-11-22T23:55:11,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711341 2024-11-22T23:55:11,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711341) roll requested 2024-11-22T23:55:11,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711348 2024-11-22T23:55:11,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711348 2024-11-22T23:55:11,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711348) roll requested 2024-11-22T23:55:11,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711356 2024-11-22T23:55:11,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711356 2024-11-22T23:55:11,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711356) roll requested 2024-11-22T23:55:11,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711363 2024-11-22T23:55:11,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711363 2024-11-22T23:55:11,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711363) roll requested 2024-11-22T23:55:11,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711370 2024-11-22T23:55:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711370 2024-11-22T23:55:11,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711370) roll requested 2024-11-22T23:55:11,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711377 2024-11-22T23:55:11,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711377 2024-11-22T23:55:11,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711377) roll requested 2024-11-22T23:55:11,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711384 2024-11-22T23:55:11,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711384 2024-11-22T23:55:11,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711384) roll requested 2024-11-22T23:55:11,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711390 2024-11-22T23:55:11,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711390 2024-11-22T23:55:11,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711390) roll requested 2024-11-22T23:55:11,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711398 2024-11-22T23:55:11,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711398 2024-11-22T23:55:11,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711398) roll requested 2024-11-22T23:55:11,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711405 2024-11-22T23:55:11,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711405 2024-11-22T23:55:11,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711405) roll requested 2024-11-22T23:55:11,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711411 2024-11-22T23:55:11,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711411 2024-11-22T23:55:11,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711411) roll requested 2024-11-22T23:55:11,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711418 2024-11-22T23:55:11,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711418 2024-11-22T23:55:11,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711418) roll requested 2024-11-22T23:55:11,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711425 2024-11-22T23:55:11,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711425 2024-11-22T23:55:11,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711425) roll requested 2024-11-22T23:55:11,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711433 2024-11-22T23:55:11,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711433 2024-11-22T23:55:11,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711433) roll requested 2024-11-22T23:55:11,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711440 2024-11-22T23:55:11,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711440 2024-11-22T23:55:11,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711440) roll requested 2024-11-22T23:55:11,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711461 2024-11-22T23:55:11,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711461 2024-11-22T23:55:11,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711461) roll requested 2024-11-22T23:55:11,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711469 2024-11-22T23:55:11,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711469 2024-11-22T23:55:11,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711469) roll requested 2024-11-22T23:55:11,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711476 2024-11-22T23:55:11,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711476 2024-11-22T23:55:11,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711476) roll requested 2024-11-22T23:55:11,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711483 2024-11-22T23:55:11,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711483 2024-11-22T23:55:11,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711483) roll requested 2024-11-22T23:55:11,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711490 2024-11-22T23:55:11,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711490 2024-11-22T23:55:11,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711490) roll requested 2024-11-22T23:55:11,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711497 2024-11-22T23:55:11,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711497 2024-11-22T23:55:11,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711497) roll requested 2024-11-22T23:55:11,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711504 2024-11-22T23:55:11,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711504 2024-11-22T23:55:11,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711504) roll requested 2024-11-22T23:55:11,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711513 2024-11-22T23:55:11,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711513 2024-11-22T23:55:11,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711513) roll requested 2024-11-22T23:55:11,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711521 2024-11-22T23:55:11,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711521 2024-11-22T23:55:11,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711521) roll requested 2024-11-22T23:55:11,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711533 2024-11-22T23:55:11,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711533 2024-11-22T23:55:11,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711533) roll requested 2024-11-22T23:55:11,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711541 2024-11-22T23:55:11,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711541 2024-11-22T23:55:11,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711541) roll requested 2024-11-22T23:55:11,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711548 2024-11-22T23:55:11,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711548 2024-11-22T23:55:11,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711548) roll requested 2024-11-22T23:55:11,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711556 2024-11-22T23:55:11,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711556 2024-11-22T23:55:11,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711556) roll requested 2024-11-22T23:55:11,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711564 2024-11-22T23:55:11,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711564 2024-11-22T23:55:11,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711564) roll requested 2024-11-22T23:55:11,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711571 2024-11-22T23:55:11,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711571 2024-11-22T23:55:11,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711571) roll requested 2024-11-22T23:55:11,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711578 2024-11-22T23:55:11,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711578 2024-11-22T23:55:11,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711578) roll requested 2024-11-22T23:55:11,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711585 2024-11-22T23:55:11,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711585 2024-11-22T23:55:11,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711585) roll requested 2024-11-22T23:55:11,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711592 2024-11-22T23:55:11,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711592 2024-11-22T23:55:11,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711592) roll requested 2024-11-22T23:55:11,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711599 2024-11-22T23:55:11,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711599 2024-11-22T23:55:11,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711599) roll requested 2024-11-22T23:55:11,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711606 2024-11-22T23:55:11,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711606 2024-11-22T23:55:11,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711606) roll requested 2024-11-22T23:55:11,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711614 2024-11-22T23:55:11,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711614 2024-11-22T23:55:11,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711614) roll requested 2024-11-22T23:55:11,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711621 2024-11-22T23:55:11,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711621 2024-11-22T23:55:11,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711621) roll requested 2024-11-22T23:55:11,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711628 2024-11-22T23:55:11,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711628 2024-11-22T23:55:11,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711628) roll requested 2024-11-22T23:55:11,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711636 2024-11-22T23:55:11,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711636 2024-11-22T23:55:11,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711636) roll requested 2024-11-22T23:55:11,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711645 2024-11-22T23:55:11,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711645 2024-11-22T23:55:11,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711645) roll requested 2024-11-22T23:55:11,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711652 2024-11-22T23:55:11,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711652 2024-11-22T23:55:11,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711652) roll requested 2024-11-22T23:55:11,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711659 2024-11-22T23:55:11,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711659 2024-11-22T23:55:11,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711659) roll requested 2024-11-22T23:55:11,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711667 2024-11-22T23:55:11,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711667 2024-11-22T23:55:11,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711667) roll requested 2024-11-22T23:55:11,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711674 2024-11-22T23:55:11,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711674 2024-11-22T23:55:11,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711674) roll requested 2024-11-22T23:55:11,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711681 2024-11-22T23:55:11,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711681 2024-11-22T23:55:11,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711681) roll requested 2024-11-22T23:55:11,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711689 2024-11-22T23:55:11,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711689 2024-11-22T23:55:11,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711689) roll requested 2024-11-22T23:55:11,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711696 2024-11-22T23:55:11,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711696 2024-11-22T23:55:11,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711696) roll requested 2024-11-22T23:55:11,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711703 2024-11-22T23:55:11,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711703 2024-11-22T23:55:11,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711703) roll requested 2024-11-22T23:55:11,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711710 2024-11-22T23:55:11,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711710 2024-11-22T23:55:11,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711710) roll requested 2024-11-22T23:55:11,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711717 2024-11-22T23:55:11,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711717 2024-11-22T23:55:11,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711717) roll requested 2024-11-22T23:55:11,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711724 2024-11-22T23:55:11,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711724 2024-11-22T23:55:11,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711724) roll requested 2024-11-22T23:55:11,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711732 2024-11-22T23:55:11,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711732 2024-11-22T23:55:11,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711732) roll requested 2024-11-22T23:55:11,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711739 2024-11-22T23:55:11,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711739 2024-11-22T23:55:11,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711739) roll requested 2024-11-22T23:55:11,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711747 2024-11-22T23:55:11,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711747 2024-11-22T23:55:11,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711747) roll requested 2024-11-22T23:55:11,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711754 2024-11-22T23:55:11,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711754 2024-11-22T23:55:11,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711754) roll requested 2024-11-22T23:55:11,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711761 2024-11-22T23:55:11,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711761 2024-11-22T23:55:11,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711761) roll requested 2024-11-22T23:55:11,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711769 2024-11-22T23:55:11,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711769 2024-11-22T23:55:11,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711769) roll requested 2024-11-22T23:55:11,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711776 2024-11-22T23:55:11,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711776 2024-11-22T23:55:11,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711776) roll requested 2024-11-22T23:55:11,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711783 2024-11-22T23:55:11,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711783 2024-11-22T23:55:11,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711783) roll requested 2024-11-22T23:55:11,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711790 2024-11-22T23:55:11,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711790 2024-11-22T23:55:11,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711790) roll requested 2024-11-22T23:55:11,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711796 2024-11-22T23:55:11,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711796 2024-11-22T23:55:11,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711796) roll requested 2024-11-22T23:55:11,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711803 2024-11-22T23:55:11,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711803 2024-11-22T23:55:11,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711803) roll requested 2024-11-22T23:55:11,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711810 2024-11-22T23:55:11,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711810 2024-11-22T23:55:11,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711810) roll requested 2024-11-22T23:55:11,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711817 2024-11-22T23:55:11,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711817 2024-11-22T23:55:11,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711817) roll requested 2024-11-22T23:55:11,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711824 2024-11-22T23:55:11,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711824 2024-11-22T23:55:11,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711824) roll requested 2024-11-22T23:55:11,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711831 2024-11-22T23:55:11,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711831 2024-11-22T23:55:11,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711831) roll requested 2024-11-22T23:55:11,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711838 2024-11-22T23:55:11,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711838 2024-11-22T23:55:11,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711838) roll requested 2024-11-22T23:55:11,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711845 2024-11-22T23:55:11,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711845 2024-11-22T23:55:11,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711845) roll requested 2024-11-22T23:55:11,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711852 2024-11-22T23:55:11,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711852 2024-11-22T23:55:11,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711852) roll requested 2024-11-22T23:55:11,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711859 2024-11-22T23:55:11,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711859 2024-11-22T23:55:11,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711859) roll requested 2024-11-22T23:55:11,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711866 2024-11-22T23:55:11,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711866 2024-11-22T23:55:11,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711866) roll requested 2024-11-22T23:55:11,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711873 2024-11-22T23:55:11,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711873 2024-11-22T23:55:11,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711873) roll requested 2024-11-22T23:55:11,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711881 2024-11-22T23:55:11,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711881 2024-11-22T23:55:11,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711881) roll requested 2024-11-22T23:55:11,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711888 2024-11-22T23:55:11,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711888 2024-11-22T23:55:11,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711888) roll requested 2024-11-22T23:55:11,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711894 2024-11-22T23:55:11,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711894 2024-11-22T23:55:11,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711894) roll requested 2024-11-22T23:55:11,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711901 2024-11-22T23:55:11,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711901 2024-11-22T23:55:11,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711901) roll requested 2024-11-22T23:55:11,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711908 2024-11-22T23:55:11,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711908 2024-11-22T23:55:11,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711908) roll requested 2024-11-22T23:55:11,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711914 2024-11-22T23:55:11,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711914 2024-11-22T23:55:11,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711914) roll requested 2024-11-22T23:55:11,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711921 2024-11-22T23:55:11,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711921 2024-11-22T23:55:11,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711921) roll requested 2024-11-22T23:55:11,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711928 2024-11-22T23:55:11,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711928 2024-11-22T23:55:11,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711928) roll requested 2024-11-22T23:55:11,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711935 2024-11-22T23:55:11,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711935 2024-11-22T23:55:11,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711935) roll requested 2024-11-22T23:55:11,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711942 2024-11-22T23:55:11,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711942 2024-11-22T23:55:11,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711942) roll requested 2024-11-22T23:55:11,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711949 2024-11-22T23:55:11,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711949 2024-11-22T23:55:11,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711949) roll requested 2024-11-22T23:55:11,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711956 2024-11-22T23:55:11,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711956 2024-11-22T23:55:11,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711956) roll requested 2024-11-22T23:55:11,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711963 2024-11-22T23:55:11,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711963 2024-11-22T23:55:11,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,970 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711963) roll requested 2024-11-22T23:55:11,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711971 2024-11-22T23:55:11,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711963 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711971 2024-11-22T23:55:11,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711971) roll requested 2024-11-22T23:55:11,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711978 2024-11-22T23:55:11,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711978 2024-11-22T23:55:11,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711978) roll requested 2024-11-22T23:55:11,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711985 2024-11-22T23:55:11,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711985 2024-11-22T23:55:11,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711985) roll requested 2024-11-22T23:55:11,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711992 2024-11-22T23:55:11,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:11,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711992 2024-11-22T23:55:11,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:11,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:11,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711992) roll requested 2024-11-22T23:55:11,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319711999 2024-11-22T23:55:12,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711999 2024-11-22T23:55:12,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319711999) roll requested 2024-11-22T23:55:12,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712006 2024-11-22T23:55:12,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319711999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712006 2024-11-22T23:55:12,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712006) roll requested 2024-11-22T23:55:12,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712012 2024-11-22T23:55:12,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712012 2024-11-22T23:55:12,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712012) roll requested 2024-11-22T23:55:12,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712019 2024-11-22T23:55:12,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712019 2024-11-22T23:55:12,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712019) roll requested 2024-11-22T23:55:12,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712026 2024-11-22T23:55:12,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712026 2024-11-22T23:55:12,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712026) roll requested 2024-11-22T23:55:12,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712034 2024-11-22T23:55:12,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712034 2024-11-22T23:55:12,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712034) roll requested 2024-11-22T23:55:12,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712041 2024-11-22T23:55:12,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712041 2024-11-22T23:55:12,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712041) roll requested 2024-11-22T23:55:12,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712048 2024-11-22T23:55:12,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712048 2024-11-22T23:55:12,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712048) roll requested 2024-11-22T23:55:12,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712055 2024-11-22T23:55:12,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712055 2024-11-22T23:55:12,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712055) roll requested 2024-11-22T23:55:12,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712062 2024-11-22T23:55:12,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712062 2024-11-22T23:55:12,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712062) roll requested 2024-11-22T23:55:12,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712088 2024-11-22T23:55:12,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712088 2024-11-22T23:55:12,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712088) roll requested 2024-11-22T23:55:12,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712101 2024-11-22T23:55:12,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712101 2024-11-22T23:55:12,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712101) roll requested 2024-11-22T23:55:12,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712108 2024-11-22T23:55:12,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712108 2024-11-22T23:55:12,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712108) roll requested 2024-11-22T23:55:12,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712116 2024-11-22T23:55:12,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712116 2024-11-22T23:55:12,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712116) roll requested 2024-11-22T23:55:12,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712123 2024-11-22T23:55:12,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712123 2024-11-22T23:55:12,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712123) roll requested 2024-11-22T23:55:12,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712130 2024-11-22T23:55:12,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712130 2024-11-22T23:55:12,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712130) roll requested 2024-11-22T23:55:12,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712136 2024-11-22T23:55:12,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712136 2024-11-22T23:55:12,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712136) roll requested 2024-11-22T23:55:12,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712142 2024-11-22T23:55:12,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712142 2024-11-22T23:55:12,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712142) roll requested 2024-11-22T23:55:12,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712149 2024-11-22T23:55:12,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712149 2024-11-22T23:55:12,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712149) roll requested 2024-11-22T23:55:12,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712156 2024-11-22T23:55:12,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712156 2024-11-22T23:55:12,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712156) roll requested 2024-11-22T23:55:12,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712162 2024-11-22T23:55:12,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712162 2024-11-22T23:55:12,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712162) roll requested 2024-11-22T23:55:12,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712169 2024-11-22T23:55:12,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712169 2024-11-22T23:55:12,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712169) roll requested 2024-11-22T23:55:12,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712176 2024-11-22T23:55:12,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712176 2024-11-22T23:55:12,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712176) roll requested 2024-11-22T23:55:12,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712183 2024-11-22T23:55:12,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712183 2024-11-22T23:55:12,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712183) roll requested 2024-11-22T23:55:12,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712190 2024-11-22T23:55:12,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712190 2024-11-22T23:55:12,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712190) roll requested 2024-11-22T23:55:12,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712197 2024-11-22T23:55:12,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712197 2024-11-22T23:55:12,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712197) roll requested 2024-11-22T23:55:12,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712203 2024-11-22T23:55:12,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712203 2024-11-22T23:55:12,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712203) roll requested 2024-11-22T23:55:12,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712210 2024-11-22T23:55:12,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712210 2024-11-22T23:55:12,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712210) roll requested 2024-11-22T23:55:12,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712217 2024-11-22T23:55:12,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712217 2024-11-22T23:55:12,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712217) roll requested 2024-11-22T23:55:12,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712224 2024-11-22T23:55:12,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712224 2024-11-22T23:55:12,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712224) roll requested 2024-11-22T23:55:12,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712231 2024-11-22T23:55:12,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712231 2024-11-22T23:55:12,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712231) roll requested 2024-11-22T23:55:12,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712238 2024-11-22T23:55:12,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712238 2024-11-22T23:55:12,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712238) roll requested 2024-11-22T23:55:12,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712244 2024-11-22T23:55:12,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712244 2024-11-22T23:55:12,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712244) roll requested 2024-11-22T23:55:12,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712252 2024-11-22T23:55:12,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712252 2024-11-22T23:55:12,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712252) roll requested 2024-11-22T23:55:12,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712259 2024-11-22T23:55:12,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712259 2024-11-22T23:55:12,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712259) roll requested 2024-11-22T23:55:12,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712266 2024-11-22T23:55:12,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712266 2024-11-22T23:55:12,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712266) roll requested 2024-11-22T23:55:12,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712273 2024-11-22T23:55:12,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712273 2024-11-22T23:55:12,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712273) roll requested 2024-11-22T23:55:12,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712280 2024-11-22T23:55:12,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712280 2024-11-22T23:55:12,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712280) roll requested 2024-11-22T23:55:12,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712287 2024-11-22T23:55:12,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712287 2024-11-22T23:55:12,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712287) roll requested 2024-11-22T23:55:12,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712294 2024-11-22T23:55:12,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712294 2024-11-22T23:55:12,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712294) roll requested 2024-11-22T23:55:12,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712300 2024-11-22T23:55:12,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712300 2024-11-22T23:55:12,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712300) roll requested 2024-11-22T23:55:12,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712307 2024-11-22T23:55:12,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712307 2024-11-22T23:55:12,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712307) roll requested 2024-11-22T23:55:12,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712314 2024-11-22T23:55:12,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712314 2024-11-22T23:55:12,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712314) roll requested 2024-11-22T23:55:12,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712320 2024-11-22T23:55:12,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712320 2024-11-22T23:55:12,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712320) roll requested 2024-11-22T23:55:12,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712327 2024-11-22T23:55:12,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712327 2024-11-22T23:55:12,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712327) roll requested 2024-11-22T23:55:12,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712334 2024-11-22T23:55:12,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712334 2024-11-22T23:55:12,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712334) roll requested 2024-11-22T23:55:12,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712341 2024-11-22T23:55:12,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712341 2024-11-22T23:55:12,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712341) roll requested 2024-11-22T23:55:12,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712348 2024-11-22T23:55:12,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712348 2024-11-22T23:55:12,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712348) roll requested 2024-11-22T23:55:12,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712355 2024-11-22T23:55:12,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712355 2024-11-22T23:55:12,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712355) roll requested 2024-11-22T23:55:12,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712361 2024-11-22T23:55:12,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712361 2024-11-22T23:55:12,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712361) roll requested 2024-11-22T23:55:12,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712369 2024-11-22T23:55:12,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712369 2024-11-22T23:55:12,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712369) roll requested 2024-11-22T23:55:12,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712376 2024-11-22T23:55:12,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712376 2024-11-22T23:55:12,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712376) roll requested 2024-11-22T23:55:12,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712382 2024-11-22T23:55:12,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712382 2024-11-22T23:55:12,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712382) roll requested 2024-11-22T23:55:12,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712389 2024-11-22T23:55:12,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712389 2024-11-22T23:55:12,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712389) roll requested 2024-11-22T23:55:12,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712396 2024-11-22T23:55:12,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712396 2024-11-22T23:55:12,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712396) roll requested 2024-11-22T23:55:12,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712402 2024-11-22T23:55:12,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712402 2024-11-22T23:55:12,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712402) roll requested 2024-11-22T23:55:12,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712409 2024-11-22T23:55:12,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712409 2024-11-22T23:55:12,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712409) roll requested 2024-11-22T23:55:12,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712416 2024-11-22T23:55:12,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712416 2024-11-22T23:55:12,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712416) roll requested 2024-11-22T23:55:12,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712422 2024-11-22T23:55:12,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712422 2024-11-22T23:55:12,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712422) roll requested 2024-11-22T23:55:12,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712429 2024-11-22T23:55:12,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712429 2024-11-22T23:55:12,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712429) roll requested 2024-11-22T23:55:12,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712437 2024-11-22T23:55:12,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712437 2024-11-22T23:55:12,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712437) roll requested 2024-11-22T23:55:12,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712443 2024-11-22T23:55:12,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712443 2024-11-22T23:55:12,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712443) roll requested 2024-11-22T23:55:12,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712450 2024-11-22T23:55:12,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712450 2024-11-22T23:55:12,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712450) roll requested 2024-11-22T23:55:12,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712457 2024-11-22T23:55:12,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712457 2024-11-22T23:55:12,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712457) roll requested 2024-11-22T23:55:12,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712464 2024-11-22T23:55:12,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712464 2024-11-22T23:55:12,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712464) roll requested 2024-11-22T23:55:12,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712471 2024-11-22T23:55:12,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712471 2024-11-22T23:55:12,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712471) roll requested 2024-11-22T23:55:12,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712477 2024-11-22T23:55:12,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712477 2024-11-22T23:55:12,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712477) roll requested 2024-11-22T23:55:12,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712484 2024-11-22T23:55:12,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712484 2024-11-22T23:55:12,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712484) roll requested 2024-11-22T23:55:12,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712492 2024-11-22T23:55:12,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712492 2024-11-22T23:55:12,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712492) roll requested 2024-11-22T23:55:12,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712498 2024-11-22T23:55:12,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712498 2024-11-22T23:55:12,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712498) roll requested 2024-11-22T23:55:12,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712505 2024-11-22T23:55:12,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712505 2024-11-22T23:55:12,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712505) roll requested 2024-11-22T23:55:12,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712512 2024-11-22T23:55:12,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712512 2024-11-22T23:55:12,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,518 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712512) roll requested 2024-11-22T23:55:12,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712519 2024-11-22T23:55:12,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712519 2024-11-22T23:55:12,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712519) roll requested 2024-11-22T23:55:12,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712526 2024-11-22T23:55:12,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712526 2024-11-22T23:55:12,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712526) roll requested 2024-11-22T23:55:12,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712534 2024-11-22T23:55:12,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712534 2024-11-22T23:55:12,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712534) roll requested 2024-11-22T23:55:12,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712542 2024-11-22T23:55:12,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712542 2024-11-22T23:55:12,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712542) roll requested 2024-11-22T23:55:12,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712548 2024-11-22T23:55:12,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712548 2024-11-22T23:55:12,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712548) roll requested 2024-11-22T23:55:12,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712556 2024-11-22T23:55:12,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712556 2024-11-22T23:55:12,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712556) roll requested 2024-11-22T23:55:12,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712564 2024-11-22T23:55:12,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712564 2024-11-22T23:55:12,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712564) roll requested 2024-11-22T23:55:12,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712571 2024-11-22T23:55:12,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712571 2024-11-22T23:55:12,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712571) roll requested 2024-11-22T23:55:12,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712578 2024-11-22T23:55:12,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712578 2024-11-22T23:55:12,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712578) roll requested 2024-11-22T23:55:12,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712586 2024-11-22T23:55:12,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712586 2024-11-22T23:55:12,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712586) roll requested 2024-11-22T23:55:12,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712593 2024-11-22T23:55:12,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712593 2024-11-22T23:55:12,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712593) roll requested 2024-11-22T23:55:12,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712600 2024-11-22T23:55:12,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712600 2024-11-22T23:55:12,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712600) roll requested 2024-11-22T23:55:12,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712608 2024-11-22T23:55:12,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712608 2024-11-22T23:55:12,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712608) roll requested 2024-11-22T23:55:12,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712625 2024-11-22T23:55:12,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712625 2024-11-22T23:55:12,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712625) roll requested 2024-11-22T23:55:12,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712637 2024-11-22T23:55:12,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712637 2024-11-22T23:55:12,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712637) roll requested 2024-11-22T23:55:12,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712648 2024-11-22T23:55:12,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712648 2024-11-22T23:55:12,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712648) roll requested 2024-11-22T23:55:12,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712656 2024-11-22T23:55:12,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712656 2024-11-22T23:55:12,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712656) roll requested 2024-11-22T23:55:12,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712662 2024-11-22T23:55:12,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712662 2024-11-22T23:55:12,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712662) roll requested 2024-11-22T23:55:12,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712669 2024-11-22T23:55:12,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712669 2024-11-22T23:55:12,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712669) roll requested 2024-11-22T23:55:12,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712676 2024-11-22T23:55:12,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712676 2024-11-22T23:55:12,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712676) roll requested 2024-11-22T23:55:12,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712682 2024-11-22T23:55:12,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712682 2024-11-22T23:55:12,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712682) roll requested 2024-11-22T23:55:12,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712689 2024-11-22T23:55:12,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712689 2024-11-22T23:55:12,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712689) roll requested 2024-11-22T23:55:12,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712696 2024-11-22T23:55:12,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712696 2024-11-22T23:55:12,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712696) roll requested 2024-11-22T23:55:12,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712702 2024-11-22T23:55:12,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712702 2024-11-22T23:55:12,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712702) roll requested 2024-11-22T23:55:12,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712709 2024-11-22T23:55:12,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712709 2024-11-22T23:55:12,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712709) roll requested 2024-11-22T23:55:12,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712715 2024-11-22T23:55:12,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712715 2024-11-22T23:55:12,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712715) roll requested 2024-11-22T23:55:12,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712722 2024-11-22T23:55:12,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712722 2024-11-22T23:55:12,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712722) roll requested 2024-11-22T23:55:12,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712729 2024-11-22T23:55:12,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712729 2024-11-22T23:55:12,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712729) roll requested 2024-11-22T23:55:12,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712736 2024-11-22T23:55:12,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712736 2024-11-22T23:55:12,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712736) roll requested 2024-11-22T23:55:12,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712742 2024-11-22T23:55:12,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712742 2024-11-22T23:55:12,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712742) roll requested 2024-11-22T23:55:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712749 2024-11-22T23:55:12,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712749 2024-11-22T23:55:12,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712749) roll requested 2024-11-22T23:55:12,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712755 2024-11-22T23:55:12,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712755 2024-11-22T23:55:12,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712755) roll requested 2024-11-22T23:55:12,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712762 2024-11-22T23:55:12,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712762 2024-11-22T23:55:12,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712762) roll requested 2024-11-22T23:55:12,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712769 2024-11-22T23:55:12,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712769 2024-11-22T23:55:12,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712769) roll requested 2024-11-22T23:55:12,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712776 2024-11-22T23:55:12,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712776 2024-11-22T23:55:12,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712776) roll requested 2024-11-22T23:55:12,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712783 2024-11-22T23:55:12,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712783 2024-11-22T23:55:12,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712783) roll requested 2024-11-22T23:55:12,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712790 2024-11-22T23:55:12,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712790 2024-11-22T23:55:12,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712790) roll requested 2024-11-22T23:55:12,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712802 2024-11-22T23:55:12,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712802 2024-11-22T23:55:12,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712802) roll requested 2024-11-22T23:55:12,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712813 2024-11-22T23:55:12,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712813 2024-11-22T23:55:12,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712813) roll requested 2024-11-22T23:55:12,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712822 2024-11-22T23:55:12,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712822 2024-11-22T23:55:12,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712822) roll requested 2024-11-22T23:55:12,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712832 2024-11-22T23:55:12,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712832 2024-11-22T23:55:12,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712832) roll requested 2024-11-22T23:55:12,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712840 2024-11-22T23:55:12,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712840 2024-11-22T23:55:12,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712840) roll requested 2024-11-22T23:55:12,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712847 2024-11-22T23:55:12,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712847 2024-11-22T23:55:12,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712847) roll requested 2024-11-22T23:55:12,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712854 2024-11-22T23:55:12,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712854 2024-11-22T23:55:12,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712854) roll requested 2024-11-22T23:55:12,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712861 2024-11-22T23:55:12,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712861 2024-11-22T23:55:12,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712861) roll requested 2024-11-22T23:55:12,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712868 2024-11-22T23:55:12,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712868 2024-11-22T23:55:12,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712868) roll requested 2024-11-22T23:55:12,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712876 2024-11-22T23:55:12,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712876 2024-11-22T23:55:12,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712876) roll requested 2024-11-22T23:55:12,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712882 2024-11-22T23:55:12,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712882 2024-11-22T23:55:12,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712882) roll requested 2024-11-22T23:55:12,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712889 2024-11-22T23:55:12,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712889 2024-11-22T23:55:12,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712889) roll requested 2024-11-22T23:55:12,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712896 2024-11-22T23:55:12,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712896 2024-11-22T23:55:12,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712896) roll requested 2024-11-22T23:55:12,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712903 2024-11-22T23:55:12,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712903 2024-11-22T23:55:12,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712903) roll requested 2024-11-22T23:55:12,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712918 2024-11-22T23:55:12,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712918 2024-11-22T23:55:12,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712918) roll requested 2024-11-22T23:55:12,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712925 2024-11-22T23:55:12,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712925 2024-11-22T23:55:12,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712925) roll requested 2024-11-22T23:55:12,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712931 2024-11-22T23:55:12,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712931 2024-11-22T23:55:12,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712931) roll requested 2024-11-22T23:55:12,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712938 2024-11-22T23:55:12,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712938 2024-11-22T23:55:12,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712938) roll requested 2024-11-22T23:55:12,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712945 2024-11-22T23:55:12,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712945 2024-11-22T23:55:12,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712945) roll requested 2024-11-22T23:55:12,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712951 2024-11-22T23:55:12,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712951 2024-11-22T23:55:12,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712951) roll requested 2024-11-22T23:55:12,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712958 2024-11-22T23:55:12,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712958 2024-11-22T23:55:12,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712958) roll requested 2024-11-22T23:55:12,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712964 2024-11-22T23:55:12,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712964 2024-11-22T23:55:12,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712964) roll requested 2024-11-22T23:55:12,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712971 2024-11-22T23:55:12,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712971 2024-11-22T23:55:12,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712971) roll requested 2024-11-22T23:55:12,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712978 2024-11-22T23:55:12,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712978 2024-11-22T23:55:12,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712978) roll requested 2024-11-22T23:55:12,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712984 2024-11-22T23:55:12,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712984 2024-11-22T23:55:12,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712984) roll requested 2024-11-22T23:55:12,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712991 2024-11-22T23:55:12,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:12,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712991 2024-11-22T23:55:12,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:12,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:12,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712991) roll requested 2024-11-22T23:55:12,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319712998 2024-11-22T23:55:13,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712998 2024-11-22T23:55:13,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319712998) roll requested 2024-11-22T23:55:13,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713004 2024-11-22T23:55:13,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319712998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713004 2024-11-22T23:55:13,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713004) roll requested 2024-11-22T23:55:13,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713011 2024-11-22T23:55:13,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713011 2024-11-22T23:55:13,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713011) roll requested 2024-11-22T23:55:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713017 2024-11-22T23:55:13,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713017 2024-11-22T23:55:13,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713017) roll requested 2024-11-22T23:55:13,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713024 2024-11-22T23:55:13,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713024 2024-11-22T23:55:13,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713024) roll requested 2024-11-22T23:55:13,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713030 2024-11-22T23:55:13,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713030 2024-11-22T23:55:13,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713030) roll requested 2024-11-22T23:55:13,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713037 2024-11-22T23:55:13,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713037 2024-11-22T23:55:13,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713037) roll requested 2024-11-22T23:55:13,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713044 2024-11-22T23:55:13,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713044 2024-11-22T23:55:13,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713044) roll requested 2024-11-22T23:55:13,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713050 2024-11-22T23:55:13,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713050 2024-11-22T23:55:13,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713050) roll requested 2024-11-22T23:55:13,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713057 2024-11-22T23:55:13,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713057 2024-11-22T23:55:13,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713057) roll requested 2024-11-22T23:55:13,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713064 2024-11-22T23:55:13,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713064 2024-11-22T23:55:13,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713064) roll requested 2024-11-22T23:55:13,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713071 2024-11-22T23:55:13,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713071 2024-11-22T23:55:13,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713071) roll requested 2024-11-22T23:55:13,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713077 2024-11-22T23:55:13,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713077 2024-11-22T23:55:13,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713077) roll requested 2024-11-22T23:55:13,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713084 2024-11-22T23:55:13,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713084 2024-11-22T23:55:13,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713084) roll requested 2024-11-22T23:55:13,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713090 2024-11-22T23:55:13,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713090 2024-11-22T23:55:13,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713090) roll requested 2024-11-22T23:55:13,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713097 2024-11-22T23:55:13,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713097 2024-11-22T23:55:13,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713097) roll requested 2024-11-22T23:55:13,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713103 2024-11-22T23:55:13,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713103 2024-11-22T23:55:13,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713103) roll requested 2024-11-22T23:55:13,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713110 2024-11-22T23:55:13,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713110 2024-11-22T23:55:13,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713110) roll requested 2024-11-22T23:55:13,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713117 2024-11-22T23:55:13,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713117 2024-11-22T23:55:13,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713117) roll requested 2024-11-22T23:55:13,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713124 2024-11-22T23:55:13,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713124 2024-11-22T23:55:13,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713124) roll requested 2024-11-22T23:55:13,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713131 2024-11-22T23:55:13,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713131 2024-11-22T23:55:13,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713131) roll requested 2024-11-22T23:55:13,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713138 2024-11-22T23:55:13,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713138 2024-11-22T23:55:13,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713138) roll requested 2024-11-22T23:55:13,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713144 2024-11-22T23:55:13,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713144 2024-11-22T23:55:13,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713144) roll requested 2024-11-22T23:55:13,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713151 2024-11-22T23:55:13,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713151 2024-11-22T23:55:13,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713151) roll requested 2024-11-22T23:55:13,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713157 2024-11-22T23:55:13,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713157 2024-11-22T23:55:13,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713157) roll requested 2024-11-22T23:55:13,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713164 2024-11-22T23:55:13,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713164 2024-11-22T23:55:13,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713164) roll requested 2024-11-22T23:55:13,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713172 2024-11-22T23:55:13,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713172 2024-11-22T23:55:13,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713172) roll requested 2024-11-22T23:55:13,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713179 2024-11-22T23:55:13,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713179 2024-11-22T23:55:13,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713179) roll requested 2024-11-22T23:55:13,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713185 2024-11-22T23:55:13,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713185 2024-11-22T23:55:13,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713185) roll requested 2024-11-22T23:55:13,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713192 2024-11-22T23:55:13,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713192 2024-11-22T23:55:13,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713192) roll requested 2024-11-22T23:55:13,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713198 2024-11-22T23:55:13,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713198 2024-11-22T23:55:13,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713198) roll requested 2024-11-22T23:55:13,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713205 2024-11-22T23:55:13,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713205 2024-11-22T23:55:13,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713205) roll requested 2024-11-22T23:55:13,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713212 2024-11-22T23:55:13,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713212 2024-11-22T23:55:13,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713212) roll requested 2024-11-22T23:55:13,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713219 2024-11-22T23:55:13,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713219 2024-11-22T23:55:13,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713219) roll requested 2024-11-22T23:55:13,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713226 2024-11-22T23:55:13,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713226 2024-11-22T23:55:13,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713226) roll requested 2024-11-22T23:55:13,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713233 2024-11-22T23:55:13,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713233 2024-11-22T23:55:13,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713233) roll requested 2024-11-22T23:55:13,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713242 2024-11-22T23:55:13,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713242 2024-11-22T23:55:13,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713242) roll requested 2024-11-22T23:55:13,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713249 2024-11-22T23:55:13,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713249 2024-11-22T23:55:13,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713249) roll requested 2024-11-22T23:55:13,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713257 2024-11-22T23:55:13,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713257 2024-11-22T23:55:13,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713257) roll requested 2024-11-22T23:55:13,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713264 2024-11-22T23:55:13,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713264 2024-11-22T23:55:13,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713264) roll requested 2024-11-22T23:55:13,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713270 2024-11-22T23:55:13,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713270 2024-11-22T23:55:13,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713270) roll requested 2024-11-22T23:55:13,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713277 2024-11-22T23:55:13,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713277 2024-11-22T23:55:13,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713277) roll requested 2024-11-22T23:55:13,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713284 2024-11-22T23:55:13,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713284 2024-11-22T23:55:13,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713284) roll requested 2024-11-22T23:55:13,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713291 2024-11-22T23:55:13,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713291 2024-11-22T23:55:13,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713291) roll requested 2024-11-22T23:55:13,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713298 2024-11-22T23:55:13,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713298 2024-11-22T23:55:13,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713298) roll requested 2024-11-22T23:55:13,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713304 2024-11-22T23:55:13,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713304 2024-11-22T23:55:13,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713304) roll requested 2024-11-22T23:55:13,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713311 2024-11-22T23:55:13,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713311 2024-11-22T23:55:13,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713311) roll requested 2024-11-22T23:55:13,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713318 2024-11-22T23:55:13,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713318 2024-11-22T23:55:13,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713318) roll requested 2024-11-22T23:55:13,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713325 2024-11-22T23:55:13,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713325 2024-11-22T23:55:13,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713325) roll requested 2024-11-22T23:55:13,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713332 2024-11-22T23:55:13,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713332 2024-11-22T23:55:13,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713332) roll requested 2024-11-22T23:55:13,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713339 2024-11-22T23:55:13,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713339 2024-11-22T23:55:13,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713339) roll requested 2024-11-22T23:55:13,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713346 2024-11-22T23:55:13,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713346 2024-11-22T23:55:13,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713346) roll requested 2024-11-22T23:55:13,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713353 2024-11-22T23:55:13,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713353 2024-11-22T23:55:13,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713353) roll requested 2024-11-22T23:55:13,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713360 2024-11-22T23:55:13,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713360 2024-11-22T23:55:13,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713360) roll requested 2024-11-22T23:55:13,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713366 2024-11-22T23:55:13,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713366 2024-11-22T23:55:13,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713366) roll requested 2024-11-22T23:55:13,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713374 2024-11-22T23:55:13,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713374 2024-11-22T23:55:13,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713374) roll requested 2024-11-22T23:55:13,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713380 2024-11-22T23:55:13,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713380 2024-11-22T23:55:13,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713380) roll requested 2024-11-22T23:55:13,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713386 2024-11-22T23:55:13,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713386 2024-11-22T23:55:13,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713386) roll requested 2024-11-22T23:55:13,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713393 2024-11-22T23:55:13,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713393 2024-11-22T23:55:13,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713393) roll requested 2024-11-22T23:55:13,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713400 2024-11-22T23:55:13,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713400 2024-11-22T23:55:13,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713400) roll requested 2024-11-22T23:55:13,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713406 2024-11-22T23:55:13,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713406 2024-11-22T23:55:13,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713406) roll requested 2024-11-22T23:55:13,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713413 2024-11-22T23:55:13,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713413 2024-11-22T23:55:13,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713413) roll requested 2024-11-22T23:55:13,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713421 2024-11-22T23:55:13,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713421 2024-11-22T23:55:13,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713421) roll requested 2024-11-22T23:55:13,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713427 2024-11-22T23:55:13,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713427 2024-11-22T23:55:13,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713427) roll requested 2024-11-22T23:55:13,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713434 2024-11-22T23:55:13,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713434 2024-11-22T23:55:13,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713434) roll requested 2024-11-22T23:55:13,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713441 2024-11-22T23:55:13,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713441 2024-11-22T23:55:13,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713441) roll requested 2024-11-22T23:55:13,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713447 2024-11-22T23:55:13,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713447 2024-11-22T23:55:13,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713447) roll requested 2024-11-22T23:55:13,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713454 2024-11-22T23:55:13,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713454 2024-11-22T23:55:13,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713454) roll requested 2024-11-22T23:55:13,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713461 2024-11-22T23:55:13,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713461 2024-11-22T23:55:13,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713461) roll requested 2024-11-22T23:55:13,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713467 2024-11-22T23:55:13,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713467 2024-11-22T23:55:13,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713467) roll requested 2024-11-22T23:55:13,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713474 2024-11-22T23:55:13,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713474 2024-11-22T23:55:13,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713474) roll requested 2024-11-22T23:55:13,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713480 2024-11-22T23:55:13,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713480 2024-11-22T23:55:13,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713480) roll requested 2024-11-22T23:55:13,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713487 2024-11-22T23:55:13,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713487 2024-11-22T23:55:13,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713487) roll requested 2024-11-22T23:55:13,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713494 2024-11-22T23:55:13,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713494 2024-11-22T23:55:13,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713494) roll requested 2024-11-22T23:55:13,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713501 2024-11-22T23:55:13,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713501 2024-11-22T23:55:13,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713501) roll requested 2024-11-22T23:55:13,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713507 2024-11-22T23:55:13,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713507 2024-11-22T23:55:13,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713507) roll requested 2024-11-22T23:55:13,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713514 2024-11-22T23:55:13,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713514 2024-11-22T23:55:13,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:13,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713514) roll requested 2024-11-22T23:55:13,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713522 2024-11-22T23:55:13,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713522 2024-11-22T23:55:13,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713522) roll requested 2024-11-22T23:55:13,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713531 2024-11-22T23:55:13,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713531 2024-11-22T23:55:13,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713531) roll requested 2024-11-22T23:55:13,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713538 2024-11-22T23:55:13,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713538 2024-11-22T23:55:13,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713538) roll requested 2024-11-22T23:55:13,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713546 2024-11-22T23:55:13,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713546 2024-11-22T23:55:13,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713546) roll requested 2024-11-22T23:55:13,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713553 2024-11-22T23:55:13,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713553 2024-11-22T23:55:13,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713553) roll requested 2024-11-22T23:55:13,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713561 2024-11-22T23:55:13,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713561 2024-11-22T23:55:13,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713561) roll requested 2024-11-22T23:55:13,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713569 2024-11-22T23:55:13,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713569 2024-11-22T23:55:13,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713569) roll requested 2024-11-22T23:55:13,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713580 2024-11-22T23:55:13,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713580 2024-11-22T23:55:13,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713580) roll requested 2024-11-22T23:55:13,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713587 2024-11-22T23:55:13,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713587 2024-11-22T23:55:13,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713587) roll requested 2024-11-22T23:55:13,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713595 2024-11-22T23:55:13,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713595 2024-11-22T23:55:13,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713595) roll requested 2024-11-22T23:55:13,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713602 2024-11-22T23:55:13,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713602 2024-11-22T23:55:13,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713602) roll requested 2024-11-22T23:55:13,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713610 2024-11-22T23:55:13,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713610 2024-11-22T23:55:13,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713610) roll requested 2024-11-22T23:55:13,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713620 2024-11-22T23:55:13,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713620 2024-11-22T23:55:13,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713620) roll requested 2024-11-22T23:55:13,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713629 2024-11-22T23:55:13,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713629 2024-11-22T23:55:13,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713629) roll requested 2024-11-22T23:55:13,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713636 2024-11-22T23:55:13,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713636 2024-11-22T23:55:13,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713636) roll requested 2024-11-22T23:55:13,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713644 2024-11-22T23:55:13,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713644 2024-11-22T23:55:13,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713644) roll requested 2024-11-22T23:55:13,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713650 2024-11-22T23:55:13,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713650 2024-11-22T23:55:13,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713650) roll requested 2024-11-22T23:55:13,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713658 2024-11-22T23:55:13,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713658 2024-11-22T23:55:13,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713658) roll requested 2024-11-22T23:55:13,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713665 2024-11-22T23:55:13,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713665 2024-11-22T23:55:13,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713665) roll requested 2024-11-22T23:55:13,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713673 2024-11-22T23:55:13,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713673 2024-11-22T23:55:13,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713673) roll requested 2024-11-22T23:55:13,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713680 2024-11-22T23:55:13,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713680 2024-11-22T23:55:13,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713680) roll requested 2024-11-22T23:55:13,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713687 2024-11-22T23:55:13,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713687 2024-11-22T23:55:13,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713687) roll requested 2024-11-22T23:55:13,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713694 2024-11-22T23:55:13,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713694 2024-11-22T23:55:13,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713694) roll requested 2024-11-22T23:55:13,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713700 2024-11-22T23:55:13,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713700 2024-11-22T23:55:13,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713700) roll requested 2024-11-22T23:55:13,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713707 2024-11-22T23:55:13,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713707 2024-11-22T23:55:13,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713707) roll requested 2024-11-22T23:55:13,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713714 2024-11-22T23:55:13,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713714 2024-11-22T23:55:13,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713714) roll requested 2024-11-22T23:55:13,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713721 2024-11-22T23:55:13,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713721 2024-11-22T23:55:13,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713721) roll requested 2024-11-22T23:55:13,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713729 2024-11-22T23:55:13,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713729 2024-11-22T23:55:13,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713729) roll requested 2024-11-22T23:55:13,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713736 2024-11-22T23:55:13,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713736 2024-11-22T23:55:13,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713736) roll requested 2024-11-22T23:55:13,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713743 2024-11-22T23:55:13,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713743 2024-11-22T23:55:13,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713743) roll requested 2024-11-22T23:55:13,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713749 2024-11-22T23:55:13,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713749 2024-11-22T23:55:13,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713749) roll requested 2024-11-22T23:55:13,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713756 2024-11-22T23:55:13,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713756 2024-11-22T23:55:13,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713756) roll requested 2024-11-22T23:55:13,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713764 2024-11-22T23:55:13,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713764 2024-11-22T23:55:13,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713764) roll requested 2024-11-22T23:55:13,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713771 2024-11-22T23:55:13,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713771 2024-11-22T23:55:13,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713771) roll requested 2024-11-22T23:55:13,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713777 2024-11-22T23:55:13,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713777 2024-11-22T23:55:13,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713777) roll requested 2024-11-22T23:55:13,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713784 2024-11-22T23:55:13,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713784 2024-11-22T23:55:13,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713784) roll requested 2024-11-22T23:55:13,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713791 2024-11-22T23:55:13,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713791 2024-11-22T23:55:13,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713791) roll requested 2024-11-22T23:55:13,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713798 2024-11-22T23:55:13,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713798 2024-11-22T23:55:13,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713798) roll requested 2024-11-22T23:55:13,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713805 2024-11-22T23:55:13,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713805 2024-11-22T23:55:13,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713805) roll requested 2024-11-22T23:55:13,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713814 2024-11-22T23:55:13,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713814 2024-11-22T23:55:13,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713814) roll requested 2024-11-22T23:55:13,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713821 2024-11-22T23:55:13,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713821 2024-11-22T23:55:13,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713821) roll requested 2024-11-22T23:55:13,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713828 2024-11-22T23:55:13,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713828 2024-11-22T23:55:13,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713828) roll requested 2024-11-22T23:55:13,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713836 2024-11-22T23:55:13,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713836 2024-11-22T23:55:13,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713836) roll requested 2024-11-22T23:55:13,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713843 2024-11-22T23:55:13,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713843 2024-11-22T23:55:13,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713843) roll requested 2024-11-22T23:55:13,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713850 2024-11-22T23:55:13,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713850 2024-11-22T23:55:13,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713850) roll requested 2024-11-22T23:55:13,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713857 2024-11-22T23:55:13,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713857 2024-11-22T23:55:13,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713857) roll requested 2024-11-22T23:55:13,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713865 2024-11-22T23:55:13,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713865 2024-11-22T23:55:13,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713865) roll requested 2024-11-22T23:55:13,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713872 2024-11-22T23:55:13,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713872 2024-11-22T23:55:13,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713872) roll requested 2024-11-22T23:55:13,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713879 2024-11-22T23:55:13,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713879 2024-11-22T23:55:13,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713879) roll requested 2024-11-22T23:55:13,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713885 2024-11-22T23:55:13,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713885 2024-11-22T23:55:13,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713885) roll requested 2024-11-22T23:55:13,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713892 2024-11-22T23:55:13,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713892 2024-11-22T23:55:13,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713892) roll requested 2024-11-22T23:55:13,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713899 2024-11-22T23:55:13,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713899 2024-11-22T23:55:13,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713899) roll requested 2024-11-22T23:55:13,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713906 2024-11-22T23:55:13,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713906 2024-11-22T23:55:13,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713906) roll requested 2024-11-22T23:55:13,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713913 2024-11-22T23:55:13,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713913 2024-11-22T23:55:13,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713913) roll requested 2024-11-22T23:55:13,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713921 2024-11-22T23:55:13,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713921 2024-11-22T23:55:13,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713921) roll requested 2024-11-22T23:55:13,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713928 2024-11-22T23:55:13,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713928 2024-11-22T23:55:13,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713928) roll requested 2024-11-22T23:55:13,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713937 2024-11-22T23:55:13,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713937 2024-11-22T23:55:13,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713937) roll requested 2024-11-22T23:55:13,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713944 2024-11-22T23:55:13,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713944 2024-11-22T23:55:13,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713944) roll requested 2024-11-22T23:55:13,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713951 2024-11-22T23:55:13,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713951 2024-11-22T23:55:13,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713951) roll requested 2024-11-22T23:55:13,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713958 2024-11-22T23:55:13,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713958 2024-11-22T23:55:13,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713958) roll requested 2024-11-22T23:55:13,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713972 2024-11-22T23:55:13,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713972 2024-11-22T23:55:13,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713972) roll requested 2024-11-22T23:55:13,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713979 2024-11-22T23:55:13,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713979 2024-11-22T23:55:13,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713979) roll requested 2024-11-22T23:55:13,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713987 2024-11-22T23:55:13,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713987 2024-11-22T23:55:13,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:13,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:13,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:13,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:13,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713987) roll requested 2024-11-22T23:55:13,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319713993 2024-11-22T23:55:13,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:13,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713993 2024-11-22T23:55:14,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319713993) roll requested 2024-11-22T23:55:14,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714000 2024-11-22T23:55:14,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319713993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714000 2024-11-22T23:55:14,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714000) roll requested 2024-11-22T23:55:14,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714007 2024-11-22T23:55:14,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714007 2024-11-22T23:55:14,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714007) roll requested 2024-11-22T23:55:14,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714014 2024-11-22T23:55:14,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714014 2024-11-22T23:55:14,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714014) roll requested 2024-11-22T23:55:14,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714022 2024-11-22T23:55:14,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714022 2024-11-22T23:55:14,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714022) roll requested 2024-11-22T23:55:14,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714029 2024-11-22T23:55:14,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714029 2024-11-22T23:55:14,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714029) roll requested 2024-11-22T23:55:14,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714035 2024-11-22T23:55:14,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714035 2024-11-22T23:55:14,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714035) roll requested 2024-11-22T23:55:14,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714042 2024-11-22T23:55:14,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714042 2024-11-22T23:55:14,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714042) roll requested 2024-11-22T23:55:14,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714049 2024-11-22T23:55:14,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714049 2024-11-22T23:55:14,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714049) roll requested 2024-11-22T23:55:14,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714056 2024-11-22T23:55:14,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714056 2024-11-22T23:55:14,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714056) roll requested 2024-11-22T23:55:14,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714063 2024-11-22T23:55:14,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714063 2024-11-22T23:55:14,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714063) roll requested 2024-11-22T23:55:14,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714081 2024-11-22T23:55:14,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714081 2024-11-22T23:55:14,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714081) roll requested 2024-11-22T23:55:14,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714089 2024-11-22T23:55:14,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714089 2024-11-22T23:55:14,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714089) roll requested 2024-11-22T23:55:14,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714096 2024-11-22T23:55:14,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714096 2024-11-22T23:55:14,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714096) roll requested 2024-11-22T23:55:14,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714103 2024-11-22T23:55:14,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714103 2024-11-22T23:55:14,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714103) roll requested 2024-11-22T23:55:14,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714110 2024-11-22T23:55:14,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714110 2024-11-22T23:55:14,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714110) roll requested 2024-11-22T23:55:14,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714117 2024-11-22T23:55:14,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714117 2024-11-22T23:55:14,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714117) roll requested 2024-11-22T23:55:14,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714124 2024-11-22T23:55:14,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714124 2024-11-22T23:55:14,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714124) roll requested 2024-11-22T23:55:14,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714131 2024-11-22T23:55:14,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714131 2024-11-22T23:55:14,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714131) roll requested 2024-11-22T23:55:14,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714138 2024-11-22T23:55:14,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714138 2024-11-22T23:55:14,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714138) roll requested 2024-11-22T23:55:14,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714145 2024-11-22T23:55:14,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714145 2024-11-22T23:55:14,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714145) roll requested 2024-11-22T23:55:14,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714152 2024-11-22T23:55:14,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714152 2024-11-22T23:55:14,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714152) roll requested 2024-11-22T23:55:14,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714160 2024-11-22T23:55:14,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714160 2024-11-22T23:55:14,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714160) roll requested 2024-11-22T23:55:14,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714172 2024-11-22T23:55:14,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714172 2024-11-22T23:55:14,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714172) roll requested 2024-11-22T23:55:14,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714181 2024-11-22T23:55:14,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714181 2024-11-22T23:55:14,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,189 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,189 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714181) roll requested 2024-11-22T23:55:14,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714190 2024-11-22T23:55:14,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714190 2024-11-22T23:55:14,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714190) roll requested 2024-11-22T23:55:14,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714197 2024-11-22T23:55:14,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714197 2024-11-22T23:55:14,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714197) roll requested 2024-11-22T23:55:14,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714204 2024-11-22T23:55:14,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714204 2024-11-22T23:55:14,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714204) roll requested 2024-11-22T23:55:14,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714211 2024-11-22T23:55:14,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714211 2024-11-22T23:55:14,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714211) roll requested 2024-11-22T23:55:14,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714218 2024-11-22T23:55:14,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714218 2024-11-22T23:55:14,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714218) roll requested 2024-11-22T23:55:14,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714225 2024-11-22T23:55:14,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714225 2024-11-22T23:55:14,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714225) roll requested 2024-11-22T23:55:14,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714232 2024-11-22T23:55:14,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714232 2024-11-22T23:55:14,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714232) roll requested 2024-11-22T23:55:14,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714239 2024-11-22T23:55:14,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714239 2024-11-22T23:55:14,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714239) roll requested 2024-11-22T23:55:14,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714246 2024-11-22T23:55:14,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714246 2024-11-22T23:55:14,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714246) roll requested 2024-11-22T23:55:14,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714254 2024-11-22T23:55:14,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714254 2024-11-22T23:55:14,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714254) roll requested 2024-11-22T23:55:14,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714261 2024-11-22T23:55:14,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714261 2024-11-22T23:55:14,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714261) roll requested 2024-11-22T23:55:14,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714268 2024-11-22T23:55:14,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714268 2024-11-22T23:55:14,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714268) roll requested 2024-11-22T23:55:14,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714276 2024-11-22T23:55:14,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714276 2024-11-22T23:55:14,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714276) roll requested 2024-11-22T23:55:14,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714283 2024-11-22T23:55:14,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714283 2024-11-22T23:55:14,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714283) roll requested 2024-11-22T23:55:14,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714289 2024-11-22T23:55:14,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714289 2024-11-22T23:55:14,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714289) roll requested 2024-11-22T23:55:14,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714296 2024-11-22T23:55:14,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714296 2024-11-22T23:55:14,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714296) roll requested 2024-11-22T23:55:14,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714309 2024-11-22T23:55:14,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714309 2024-11-22T23:55:14,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714309) roll requested 2024-11-22T23:55:14,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714316 2024-11-22T23:55:14,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714316 2024-11-22T23:55:14,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714316) roll requested 2024-11-22T23:55:14,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714323 2024-11-22T23:55:14,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714323 2024-11-22T23:55:14,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714323) roll requested 2024-11-22T23:55:14,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714330 2024-11-22T23:55:14,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714330 2024-11-22T23:55:14,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714330) roll requested 2024-11-22T23:55:14,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714337 2024-11-22T23:55:14,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714337 2024-11-22T23:55:14,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714337) roll requested 2024-11-22T23:55:14,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714343 2024-11-22T23:55:14,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714343 2024-11-22T23:55:14,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714343) roll requested 2024-11-22T23:55:14,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714351 2024-11-22T23:55:14,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714351 2024-11-22T23:55:14,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714351) roll requested 2024-11-22T23:55:14,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714357 2024-11-22T23:55:14,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714357 2024-11-22T23:55:14,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714357) roll requested 2024-11-22T23:55:14,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714364 2024-11-22T23:55:14,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714364 2024-11-22T23:55:14,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714364) roll requested 2024-11-22T23:55:14,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714371 2024-11-22T23:55:14,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714371 2024-11-22T23:55:14,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714371) roll requested 2024-11-22T23:55:14,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714378 2024-11-22T23:55:14,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714378 2024-11-22T23:55:14,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714378) roll requested 2024-11-22T23:55:14,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714384 2024-11-22T23:55:14,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714384 2024-11-22T23:55:14,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714384) roll requested 2024-11-22T23:55:14,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714391 2024-11-22T23:55:14,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714391 2024-11-22T23:55:14,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714391) roll requested 2024-11-22T23:55:14,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714398 2024-11-22T23:55:14,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714398 2024-11-22T23:55:14,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714398) roll requested 2024-11-22T23:55:14,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714405 2024-11-22T23:55:14,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714405 2024-11-22T23:55:14,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714405) roll requested 2024-11-22T23:55:14,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714411 2024-11-22T23:55:14,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714411 2024-11-22T23:55:14,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714411) roll requested 2024-11-22T23:55:14,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714418 2024-11-22T23:55:14,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714418 2024-11-22T23:55:14,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714418) roll requested 2024-11-22T23:55:14,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714425 2024-11-22T23:55:14,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714425 2024-11-22T23:55:14,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714425) roll requested 2024-11-22T23:55:14,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714433 2024-11-22T23:55:14,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714433 2024-11-22T23:55:14,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714433) roll requested 2024-11-22T23:55:14,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714440 2024-11-22T23:55:14,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714440 2024-11-22T23:55:14,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714440) roll requested 2024-11-22T23:55:14,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714447 2024-11-22T23:55:14,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714447 2024-11-22T23:55:14,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714447) roll requested 2024-11-22T23:55:14,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714454 2024-11-22T23:55:14,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714454 2024-11-22T23:55:14,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714454) roll requested 2024-11-22T23:55:14,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714461 2024-11-22T23:55:14,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714461 2024-11-22T23:55:14,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714461) roll requested 2024-11-22T23:55:14,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714468 2024-11-22T23:55:14,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714468 2024-11-22T23:55:14,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714468) roll requested 2024-11-22T23:55:14,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714478 2024-11-22T23:55:14,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714478 2024-11-22T23:55:14,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714478) roll requested 2024-11-22T23:55:14,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714486 2024-11-22T23:55:14,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714486 2024-11-22T23:55:14,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714486) roll requested 2024-11-22T23:55:14,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714493 2024-11-22T23:55:14,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714493 2024-11-22T23:55:14,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714493) roll requested 2024-11-22T23:55:14,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714500 2024-11-22T23:55:14,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714500 2024-11-22T23:55:14,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714500) roll requested 2024-11-22T23:55:14,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714506 2024-11-22T23:55:14,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714506 2024-11-22T23:55:14,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714506) roll requested 2024-11-22T23:55:14,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714513 2024-11-22T23:55:14,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714513 2024-11-22T23:55:14,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714513) roll requested 2024-11-22T23:55:14,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714521 2024-11-22T23:55:14,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714521 2024-11-22T23:55:14,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714521) roll requested 2024-11-22T23:55:14,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714527 2024-11-22T23:55:14,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714527 2024-11-22T23:55:14,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714527) roll requested 2024-11-22T23:55:14,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714534 2024-11-22T23:55:14,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714534 2024-11-22T23:55:14,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714534) roll requested 2024-11-22T23:55:14,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714541 2024-11-22T23:55:14,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714541 2024-11-22T23:55:14,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714541) roll requested 2024-11-22T23:55:14,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714548 2024-11-22T23:55:14,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714548 2024-11-22T23:55:14,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714548) roll requested 2024-11-22T23:55:14,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714557 2024-11-22T23:55:14,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714557 2024-11-22T23:55:14,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714557) roll requested 2024-11-22T23:55:14,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714569 2024-11-22T23:55:14,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714569 2024-11-22T23:55:14,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714569) roll requested 2024-11-22T23:55:14,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714577 2024-11-22T23:55:14,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714577 2024-11-22T23:55:14,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714577) roll requested 2024-11-22T23:55:14,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714584 2024-11-22T23:55:14,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714584 2024-11-22T23:55:14,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714584) roll requested 2024-11-22T23:55:14,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714591 2024-11-22T23:55:14,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714591 2024-11-22T23:55:14,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714591) roll requested 2024-11-22T23:55:14,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714598 2024-11-22T23:55:14,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714598 2024-11-22T23:55:14,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714598) roll requested 2024-11-22T23:55:14,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714608 2024-11-22T23:55:14,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714608 2024-11-22T23:55:14,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714608) roll requested 2024-11-22T23:55:14,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714615 2024-11-22T23:55:14,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714615 2024-11-22T23:55:14,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714615) roll requested 2024-11-22T23:55:14,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714623 2024-11-22T23:55:14,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714623 2024-11-22T23:55:14,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714623) roll requested 2024-11-22T23:55:14,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714630 2024-11-22T23:55:14,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714630 2024-11-22T23:55:14,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714630) roll requested 2024-11-22T23:55:14,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714637 2024-11-22T23:55:14,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714637 2024-11-22T23:55:14,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714637) roll requested 2024-11-22T23:55:14,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714644 2024-11-22T23:55:14,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714644 2024-11-22T23:55:14,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714644) roll requested 2024-11-22T23:55:14,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714651 2024-11-22T23:55:14,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714651 2024-11-22T23:55:14,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714651) roll requested 2024-11-22T23:55:14,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714658 2024-11-22T23:55:14,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714658 2024-11-22T23:55:14,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714658) roll requested 2024-11-22T23:55:14,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714665 2024-11-22T23:55:14,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714665 2024-11-22T23:55:14,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714665) roll requested 2024-11-22T23:55:14,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714673 2024-11-22T23:55:14,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714673 2024-11-22T23:55:14,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714673) roll requested 2024-11-22T23:55:14,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714679 2024-11-22T23:55:14,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714679 2024-11-22T23:55:14,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714679) roll requested 2024-11-22T23:55:14,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714686 2024-11-22T23:55:14,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714686 2024-11-22T23:55:14,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714686) roll requested 2024-11-22T23:55:14,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714693 2024-11-22T23:55:14,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714693 2024-11-22T23:55:14,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714693) roll requested 2024-11-22T23:55:14,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714700 2024-11-22T23:55:14,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714700 2024-11-22T23:55:14,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714700) roll requested 2024-11-22T23:55:14,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714707 2024-11-22T23:55:14,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714707 2024-11-22T23:55:14,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714707) roll requested 2024-11-22T23:55:14,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714715 2024-11-22T23:55:14,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714715 2024-11-22T23:55:14,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714715) roll requested 2024-11-22T23:55:14,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714722 2024-11-22T23:55:14,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714722 2024-11-22T23:55:14,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714722) roll requested 2024-11-22T23:55:14,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714730 2024-11-22T23:55:14,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714730 2024-11-22T23:55:14,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714730) roll requested 2024-11-22T23:55:14,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714736 2024-11-22T23:55:14,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714736 2024-11-22T23:55:14,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714736) roll requested 2024-11-22T23:55:14,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714744 2024-11-22T23:55:14,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714744 2024-11-22T23:55:14,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714744) roll requested 2024-11-22T23:55:14,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714752 2024-11-22T23:55:14,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714752 2024-11-22T23:55:14,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714752) roll requested 2024-11-22T23:55:14,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714759 2024-11-22T23:55:14,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714759 2024-11-22T23:55:14,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714759) roll requested 2024-11-22T23:55:14,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714766 2024-11-22T23:55:14,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714766 2024-11-22T23:55:14,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714766) roll requested 2024-11-22T23:55:14,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714773 2024-11-22T23:55:14,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714773 2024-11-22T23:55:14,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714773) roll requested 2024-11-22T23:55:14,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714779 2024-11-22T23:55:14,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714779 2024-11-22T23:55:14,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714779) roll requested 2024-11-22T23:55:14,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714786 2024-11-22T23:55:14,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714786 2024-11-22T23:55:14,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714786) roll requested 2024-11-22T23:55:14,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714793 2024-11-22T23:55:14,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714793 2024-11-22T23:55:14,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714793) roll requested 2024-11-22T23:55:14,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714800 2024-11-22T23:55:14,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714800 2024-11-22T23:55:14,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714800) roll requested 2024-11-22T23:55:14,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714806 2024-11-22T23:55:14,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714806 2024-11-22T23:55:14,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714806) roll requested 2024-11-22T23:55:14,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714813 2024-11-22T23:55:14,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714813 2024-11-22T23:55:14,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714813) roll requested 2024-11-22T23:55:14,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714820 2024-11-22T23:55:14,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714820 2024-11-22T23:55:14,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714820) roll requested 2024-11-22T23:55:14,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714828 2024-11-22T23:55:14,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714828 2024-11-22T23:55:14,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714828) roll requested 2024-11-22T23:55:14,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714835 2024-11-22T23:55:14,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714835 2024-11-22T23:55:14,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714835) roll requested 2024-11-22T23:55:14,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714842 2024-11-22T23:55:14,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714842 2024-11-22T23:55:14,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714842) roll requested 2024-11-22T23:55:14,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714849 2024-11-22T23:55:14,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714849 2024-11-22T23:55:14,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714849) roll requested 2024-11-22T23:55:14,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714857 2024-11-22T23:55:14,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714857 2024-11-22T23:55:14,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714857) roll requested 2024-11-22T23:55:14,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714868 2024-11-22T23:55:14,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714868 2024-11-22T23:55:14,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714868) roll requested 2024-11-22T23:55:14,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714882 2024-11-22T23:55:14,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714882 2024-11-22T23:55:14,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714882) roll requested 2024-11-22T23:55:14,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714889 2024-11-22T23:55:14,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714889 2024-11-22T23:55:14,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714889) roll requested 2024-11-22T23:55:14,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714897 2024-11-22T23:55:14,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714897 2024-11-22T23:55:14,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714897) roll requested 2024-11-22T23:55:14,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714903 2024-11-22T23:55:14,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714903 2024-11-22T23:55:14,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714903) roll requested 2024-11-22T23:55:14,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714910 2024-11-22T23:55:14,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714910 2024-11-22T23:55:14,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714910) roll requested 2024-11-22T23:55:14,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714917 2024-11-22T23:55:14,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714917 2024-11-22T23:55:14,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714917) roll requested 2024-11-22T23:55:14,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714923 2024-11-22T23:55:14,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714923 2024-11-22T23:55:14,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714923) roll requested 2024-11-22T23:55:14,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714936 2024-11-22T23:55:14,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714936 2024-11-22T23:55:14,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714936) roll requested 2024-11-22T23:55:14,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714943 2024-11-22T23:55:14,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714943 2024-11-22T23:55:14,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714943) roll requested 2024-11-22T23:55:14,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714953 2024-11-22T23:55:14,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714953 2024-11-22T23:55:14,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714953) roll requested 2024-11-22T23:55:14,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714960 2024-11-22T23:55:14,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714960 2024-11-22T23:55:14,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714960) roll requested 2024-11-22T23:55:14,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714968 2024-11-22T23:55:14,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714968 2024-11-22T23:55:14,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714968) roll requested 2024-11-22T23:55:14,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714974 2024-11-22T23:55:14,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714974 2024-11-22T23:55:14,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714974) roll requested 2024-11-22T23:55:14,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714981 2024-11-22T23:55:14,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714981 2024-11-22T23:55:14,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714981) roll requested 2024-11-22T23:55:14,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714988 2024-11-22T23:55:14,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:14,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714988 2024-11-22T23:55:14,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:14,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:14,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:14,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:14,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714988) roll requested 2024-11-22T23:55:14,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319714995 2024-11-22T23:55:15,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714995 2024-11-22T23:55:15,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319714995) roll requested 2024-11-22T23:55:15,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715002 2024-11-22T23:55:15,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319714995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715002 2024-11-22T23:55:15,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715002) roll requested 2024-11-22T23:55:15,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715008 2024-11-22T23:55:15,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715008 2024-11-22T23:55:15,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715008) roll requested 2024-11-22T23:55:15,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715015 2024-11-22T23:55:15,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715015 2024-11-22T23:55:15,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715015) roll requested 2024-11-22T23:55:15,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715021 2024-11-22T23:55:15,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715021 2024-11-22T23:55:15,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715021) roll requested 2024-11-22T23:55:15,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715028 2024-11-22T23:55:15,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715028 2024-11-22T23:55:15,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715028) roll requested 2024-11-22T23:55:15,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715035 2024-11-22T23:55:15,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715035 2024-11-22T23:55:15,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715035) roll requested 2024-11-22T23:55:15,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715042 2024-11-22T23:55:15,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715042 2024-11-22T23:55:15,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715042) roll requested 2024-11-22T23:55:15,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715049 2024-11-22T23:55:15,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715049 2024-11-22T23:55:15,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715049) roll requested 2024-11-22T23:55:15,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715055 2024-11-22T23:55:15,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715055 2024-11-22T23:55:15,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715055) roll requested 2024-11-22T23:55:15,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715062 2024-11-22T23:55:15,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715062 2024-11-22T23:55:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715062) roll requested 2024-11-22T23:55:15,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715069 2024-11-22T23:55:15,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715069 2024-11-22T23:55:15,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715069) roll requested 2024-11-22T23:55:15,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715076 2024-11-22T23:55:15,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715076 2024-11-22T23:55:15,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715076) roll requested 2024-11-22T23:55:15,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715083 2024-11-22T23:55:15,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715083 2024-11-22T23:55:15,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715083) roll requested 2024-11-22T23:55:15,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715090 2024-11-22T23:55:15,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715090 2024-11-22T23:55:15,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715090) roll requested 2024-11-22T23:55:15,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715097 2024-11-22T23:55:15,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715097 2024-11-22T23:55:15,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715097) roll requested 2024-11-22T23:55:15,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715104 2024-11-22T23:55:15,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715104 2024-11-22T23:55:15,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715104) roll requested 2024-11-22T23:55:15,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715112 2024-11-22T23:55:15,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715112 2024-11-22T23:55:15,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715112) roll requested 2024-11-22T23:55:15,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715119 2024-11-22T23:55:15,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715119 2024-11-22T23:55:15,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715119) roll requested 2024-11-22T23:55:15,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715126 2024-11-22T23:55:15,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715126 2024-11-22T23:55:15,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715126) roll requested 2024-11-22T23:55:15,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715133 2024-11-22T23:55:15,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715133 2024-11-22T23:55:15,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715133) roll requested 2024-11-22T23:55:15,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715141 2024-11-22T23:55:15,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715141 2024-11-22T23:55:15,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715141) roll requested 2024-11-22T23:55:15,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715148 2024-11-22T23:55:15,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715148 2024-11-22T23:55:15,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715148) roll requested 2024-11-22T23:55:15,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715154 2024-11-22T23:55:15,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715154 2024-11-22T23:55:15,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715154) roll requested 2024-11-22T23:55:15,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715161 2024-11-22T23:55:15,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715161 2024-11-22T23:55:15,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715161) roll requested 2024-11-22T23:55:15,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715170 2024-11-22T23:55:15,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715170 2024-11-22T23:55:15,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715170) roll requested 2024-11-22T23:55:15,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715176 2024-11-22T23:55:15,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715176 2024-11-22T23:55:15,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715176) roll requested 2024-11-22T23:55:15,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715197 2024-11-22T23:55:15,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715197 2024-11-22T23:55:15,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715197) roll requested 2024-11-22T23:55:15,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715203 2024-11-22T23:55:15,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715203 2024-11-22T23:55:15,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715203) roll requested 2024-11-22T23:55:15,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715210 2024-11-22T23:55:15,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715210 2024-11-22T23:55:15,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715210) roll requested 2024-11-22T23:55:15,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715217 2024-11-22T23:55:15,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715217 2024-11-22T23:55:15,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715217) roll requested 2024-11-22T23:55:15,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715224 2024-11-22T23:55:15,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715224 2024-11-22T23:55:15,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715224) roll requested 2024-11-22T23:55:15,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715230 2024-11-22T23:55:15,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715230 2024-11-22T23:55:15,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715230) roll requested 2024-11-22T23:55:15,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715237 2024-11-22T23:55:15,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715237 2024-11-22T23:55:15,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715237) roll requested 2024-11-22T23:55:15,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715244 2024-11-22T23:55:15,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715244 2024-11-22T23:55:15,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715244) roll requested 2024-11-22T23:55:15,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715251 2024-11-22T23:55:15,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715251 2024-11-22T23:55:15,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715251) roll requested 2024-11-22T23:55:15,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715257 2024-11-22T23:55:15,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715257 2024-11-22T23:55:15,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715257) roll requested 2024-11-22T23:55:15,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715264 2024-11-22T23:55:15,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715264 2024-11-22T23:55:15,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715264) roll requested 2024-11-22T23:55:15,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715271 2024-11-22T23:55:15,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715271 2024-11-22T23:55:15,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715271) roll requested 2024-11-22T23:55:15,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715278 2024-11-22T23:55:15,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715278 2024-11-22T23:55:15,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715278) roll requested 2024-11-22T23:55:15,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715284 2024-11-22T23:55:15,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715284 2024-11-22T23:55:15,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715284) roll requested 2024-11-22T23:55:15,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715291 2024-11-22T23:55:15,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715291 2024-11-22T23:55:15,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715291) roll requested 2024-11-22T23:55:15,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715299 2024-11-22T23:55:15,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715299 2024-11-22T23:55:15,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715299) roll requested 2024-11-22T23:55:15,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715305 2024-11-22T23:55:15,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715305 2024-11-22T23:55:15,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715305) roll requested 2024-11-22T23:55:15,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715312 2024-11-22T23:55:15,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715312 2024-11-22T23:55:15,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715312) roll requested 2024-11-22T23:55:15,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715325 2024-11-22T23:55:15,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715325 2024-11-22T23:55:15,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715325) roll requested 2024-11-22T23:55:15,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715333 2024-11-22T23:55:15,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715333 2024-11-22T23:55:15,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715333) roll requested 2024-11-22T23:55:15,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715340 2024-11-22T23:55:15,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715340 2024-11-22T23:55:15,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715340) roll requested 2024-11-22T23:55:15,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715349 2024-11-22T23:55:15,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715349 2024-11-22T23:55:15,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715349) roll requested 2024-11-22T23:55:15,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715356 2024-11-22T23:55:15,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715356 2024-11-22T23:55:15,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715356) roll requested 2024-11-22T23:55:15,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715365 2024-11-22T23:55:15,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715365 2024-11-22T23:55:15,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715365) roll requested 2024-11-22T23:55:15,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715373 2024-11-22T23:55:15,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715373 2024-11-22T23:55:15,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715373) roll requested 2024-11-22T23:55:15,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715380 2024-11-22T23:55:15,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715380 2024-11-22T23:55:15,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715380) roll requested 2024-11-22T23:55:15,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715388 2024-11-22T23:55:15,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715388 2024-11-22T23:55:15,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715388) roll requested 2024-11-22T23:55:15,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715395 2024-11-22T23:55:15,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715395 2024-11-22T23:55:15,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715395) roll requested 2024-11-22T23:55:15,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715402 2024-11-22T23:55:15,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715402 2024-11-22T23:55:15,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715402) roll requested 2024-11-22T23:55:15,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715409 2024-11-22T23:55:15,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715409 2024-11-22T23:55:15,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715409) roll requested 2024-11-22T23:55:15,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715421 2024-11-22T23:55:15,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715421 2024-11-22T23:55:15,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715421) roll requested 2024-11-22T23:55:15,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715429 2024-11-22T23:55:15,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715429 2024-11-22T23:55:15,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715429) roll requested 2024-11-22T23:55:15,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715437 2024-11-22T23:55:15,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715437 2024-11-22T23:55:15,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715437) roll requested 2024-11-22T23:55:15,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715445 2024-11-22T23:55:15,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715445 2024-11-22T23:55:15,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715445) roll requested 2024-11-22T23:55:15,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715452 2024-11-22T23:55:15,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715452 2024-11-22T23:55:15,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715452) roll requested 2024-11-22T23:55:15,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715459 2024-11-22T23:55:15,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715459 2024-11-22T23:55:15,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715459) roll requested 2024-11-22T23:55:15,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715466 2024-11-22T23:55:15,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715466 2024-11-22T23:55:15,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715466) roll requested 2024-11-22T23:55:15,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715473 2024-11-22T23:55:15,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715473 2024-11-22T23:55:15,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715473) roll requested 2024-11-22T23:55:15,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715480 2024-11-22T23:55:15,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715480 2024-11-22T23:55:15,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715480) roll requested 2024-11-22T23:55:15,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715487 2024-11-22T23:55:15,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715487 2024-11-22T23:55:15,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715487) roll requested 2024-11-22T23:55:15,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715493 2024-11-22T23:55:15,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715493 2024-11-22T23:55:15,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,500 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715493) roll requested 2024-11-22T23:55:15,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715500 2024-11-22T23:55:15,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715500 2024-11-22T23:55:15,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715500) roll requested 2024-11-22T23:55:15,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715507 2024-11-22T23:55:15,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715500 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715507 2024-11-22T23:55:15,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715507) roll requested 2024-11-22T23:55:15,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715514 2024-11-22T23:55:15,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715514 2024-11-22T23:55:15,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715514) roll requested 2024-11-22T23:55:15,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715520 2024-11-22T23:55:15,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715520 2024-11-22T23:55:15,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715520) roll requested 2024-11-22T23:55:15,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715527 2024-11-22T23:55:15,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715527 2024-11-22T23:55:15,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715527) roll requested 2024-11-22T23:55:15,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715534 2024-11-22T23:55:15,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715534 2024-11-22T23:55:15,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715534) roll requested 2024-11-22T23:55:15,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715542 2024-11-22T23:55:15,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715542 2024-11-22T23:55:15,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715542) roll requested 2024-11-22T23:55:15,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715550 2024-11-22T23:55:15,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715550 2024-11-22T23:55:15,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715550) roll requested 2024-11-22T23:55:15,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715557 2024-11-22T23:55:15,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715557 2024-11-22T23:55:15,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715557) roll requested 2024-11-22T23:55:15,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715564 2024-11-22T23:55:15,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715564 2024-11-22T23:55:15,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715564) roll requested 2024-11-22T23:55:15,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715571 2024-11-22T23:55:15,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715571 2024-11-22T23:55:15,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715571) roll requested 2024-11-22T23:55:15,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715578 2024-11-22T23:55:15,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715578 2024-11-22T23:55:15,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715578) roll requested 2024-11-22T23:55:15,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715585 2024-11-22T23:55:15,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715585 2024-11-22T23:55:15,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715585) roll requested 2024-11-22T23:55:15,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715592 2024-11-22T23:55:15,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715592 2024-11-22T23:55:15,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715592) roll requested 2024-11-22T23:55:15,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715599 2024-11-22T23:55:15,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715599 2024-11-22T23:55:15,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715599) roll requested 2024-11-22T23:55:15,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715605 2024-11-22T23:55:15,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715605 2024-11-22T23:55:15,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715605) roll requested 2024-11-22T23:55:15,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715615 2024-11-22T23:55:15,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715615 2024-11-22T23:55:15,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715615) roll requested 2024-11-22T23:55:15,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715622 2024-11-22T23:55:15,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715622 2024-11-22T23:55:15,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715622) roll requested 2024-11-22T23:55:15,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715629 2024-11-22T23:55:15,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715629 2024-11-22T23:55:15,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715629) roll requested 2024-11-22T23:55:15,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715635 2024-11-22T23:55:15,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715635 2024-11-22T23:55:15,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715635) roll requested 2024-11-22T23:55:15,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715642 2024-11-22T23:55:15,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715642 2024-11-22T23:55:15,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715642) roll requested 2024-11-22T23:55:15,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715649 2024-11-22T23:55:15,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715649 2024-11-22T23:55:15,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715649) roll requested 2024-11-22T23:55:15,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715656 2024-11-22T23:55:15,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715656 2024-11-22T23:55:15,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715656) roll requested 2024-11-22T23:55:15,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715663 2024-11-22T23:55:15,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715663 2024-11-22T23:55:15,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715663) roll requested 2024-11-22T23:55:15,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715670 2024-11-22T23:55:15,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715670 2024-11-22T23:55:15,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715670) roll requested 2024-11-22T23:55:15,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715689 2024-11-22T23:55:15,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715689 2024-11-22T23:55:15,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715689) roll requested 2024-11-22T23:55:15,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715702 2024-11-22T23:55:15,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715702 2024-11-22T23:55:15,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715702) roll requested 2024-11-22T23:55:15,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715712 2024-11-22T23:55:15,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715712 2024-11-22T23:55:15,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715712) roll requested 2024-11-22T23:55:15,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715720 2024-11-22T23:55:15,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715720 2024-11-22T23:55:15,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715720) roll requested 2024-11-22T23:55:15,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715728 2024-11-22T23:55:15,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715728 2024-11-22T23:55:15,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715728) roll requested 2024-11-22T23:55:15,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715740 2024-11-22T23:55:15,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715740 2024-11-22T23:55:15,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715740) roll requested 2024-11-22T23:55:15,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715748 2024-11-22T23:55:15,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715748 2024-11-22T23:55:15,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715748) roll requested 2024-11-22T23:55:15,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715755 2024-11-22T23:55:15,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715755 2024-11-22T23:55:15,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715755) roll requested 2024-11-22T23:55:15,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715762 2024-11-22T23:55:15,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715762 2024-11-22T23:55:15,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715762) roll requested 2024-11-22T23:55:15,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715772 2024-11-22T23:55:15,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715772 2024-11-22T23:55:15,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715772) roll requested 2024-11-22T23:55:15,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715781 2024-11-22T23:55:15,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715781 2024-11-22T23:55:15,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715781) roll requested 2024-11-22T23:55:15,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715789 2024-11-22T23:55:15,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715789 2024-11-22T23:55:15,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715789) roll requested 2024-11-22T23:55:15,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715796 2024-11-22T23:55:15,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715796 2024-11-22T23:55:15,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715796) roll requested 2024-11-22T23:55:15,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715803 2024-11-22T23:55:15,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715803 2024-11-22T23:55:15,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715803) roll requested 2024-11-22T23:55:15,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715810 2024-11-22T23:55:15,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715810 2024-11-22T23:55:15,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715810) roll requested 2024-11-22T23:55:15,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715817 2024-11-22T23:55:15,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715817 2024-11-22T23:55:15,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715817) roll requested 2024-11-22T23:55:15,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715825 2024-11-22T23:55:15,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715825 2024-11-22T23:55:15,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715825) roll requested 2024-11-22T23:55:15,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715833 2024-11-22T23:55:15,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715833 2024-11-22T23:55:15,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715833) roll requested 2024-11-22T23:55:15,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715840 2024-11-22T23:55:15,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715840 2024-11-22T23:55:15,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715840) roll requested 2024-11-22T23:55:15,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715846 2024-11-22T23:55:15,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715846 2024-11-22T23:55:15,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715846) roll requested 2024-11-22T23:55:15,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715857 2024-11-22T23:55:15,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715857 2024-11-22T23:55:15,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715857) roll requested 2024-11-22T23:55:15,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715865 2024-11-22T23:55:15,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715865 2024-11-22T23:55:15,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715865) roll requested 2024-11-22T23:55:15,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715873 2024-11-22T23:55:15,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715873 2024-11-22T23:55:15,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715873) roll requested 2024-11-22T23:55:15,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715880 2024-11-22T23:55:15,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715880 2024-11-22T23:55:15,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715880) roll requested 2024-11-22T23:55:15,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715888 2024-11-22T23:55:15,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715888 2024-11-22T23:55:15,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715888) roll requested 2024-11-22T23:55:15,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715896 2024-11-22T23:55:15,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715896 2024-11-22T23:55:15,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715896) roll requested 2024-11-22T23:55:15,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715904 2024-11-22T23:55:15,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715904 2024-11-22T23:55:15,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715904) roll requested 2024-11-22T23:55:15,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715911 2024-11-22T23:55:15,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715911 2024-11-22T23:55:15,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715911) roll requested 2024-11-22T23:55:15,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715918 2024-11-22T23:55:15,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715918 2024-11-22T23:55:15,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715918) roll requested 2024-11-22T23:55:15,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715925 2024-11-22T23:55:15,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715925 2024-11-22T23:55:15,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715925) roll requested 2024-11-22T23:55:15,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715933 2024-11-22T23:55:15,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715933 2024-11-22T23:55:15,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715933) roll requested 2024-11-22T23:55:15,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715941 2024-11-22T23:55:15,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715941 2024-11-22T23:55:15,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715941) roll requested 2024-11-22T23:55:15,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715948 2024-11-22T23:55:15,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715948 2024-11-22T23:55:15,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715948) roll requested 2024-11-22T23:55:15,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715955 2024-11-22T23:55:15,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715955 2024-11-22T23:55:15,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,962 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715955) roll requested 2024-11-22T23:55:15,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715962 2024-11-22T23:55:15,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715962 2024-11-22T23:55:15,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715962) roll requested 2024-11-22T23:55:15,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715969 2024-11-22T23:55:15,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715969 2024-11-22T23:55:15,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715969) roll requested 2024-11-22T23:55:15,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715975 2024-11-22T23:55:15,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715975 2024-11-22T23:55:15,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715975) roll requested 2024-11-22T23:55:15,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715983 2024-11-22T23:55:15,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715983 2024-11-22T23:55:15,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715983) roll requested 2024-11-22T23:55:15,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715989 2024-11-22T23:55:15,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:15,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715989 2024-11-22T23:55:15,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:15,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:15,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:15,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:15,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715989) roll requested 2024-11-22T23:55:15,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319715996 2024-11-22T23:55:16,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715996 2024-11-22T23:55:16,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319715996) roll requested 2024-11-22T23:55:16,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716003 2024-11-22T23:55:16,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319715996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716003 2024-11-22T23:55:16,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716003) roll requested 2024-11-22T23:55:16,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716010 2024-11-22T23:55:16,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716010 2024-11-22T23:55:16,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716010) roll requested 2024-11-22T23:55:16,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716017 2024-11-22T23:55:16,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716017 2024-11-22T23:55:16,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716017) roll requested 2024-11-22T23:55:16,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716023 2024-11-22T23:55:16,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716023 2024-11-22T23:55:16,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716023) roll requested 2024-11-22T23:55:16,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716030 2024-11-22T23:55:16,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716030 2024-11-22T23:55:16,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716030) roll requested 2024-11-22T23:55:16,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716044 2024-11-22T23:55:16,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716044 2024-11-22T23:55:16,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716044) roll requested 2024-11-22T23:55:16,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716051 2024-11-22T23:55:16,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716051 2024-11-22T23:55:16,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716051) roll requested 2024-11-22T23:55:16,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716058 2024-11-22T23:55:16,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716058 2024-11-22T23:55:16,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716058) roll requested 2024-11-22T23:55:16,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716064 2024-11-22T23:55:16,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716064 2024-11-22T23:55:16,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716064) roll requested 2024-11-22T23:55:16,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716071 2024-11-22T23:55:16,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716071 2024-11-22T23:55:16,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716071) roll requested 2024-11-22T23:55:16,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716078 2024-11-22T23:55:16,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716078 2024-11-22T23:55:16,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716078) roll requested 2024-11-22T23:55:16,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716084 2024-11-22T23:55:16,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716084 2024-11-22T23:55:16,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716084) roll requested 2024-11-22T23:55:16,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716091 2024-11-22T23:55:16,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716091 2024-11-22T23:55:16,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716091) roll requested 2024-11-22T23:55:16,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716099 2024-11-22T23:55:16,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716099 2024-11-22T23:55:16,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716099) roll requested 2024-11-22T23:55:16,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716106 2024-11-22T23:55:16,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716106 2024-11-22T23:55:16,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,113 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716106) roll requested 2024-11-22T23:55:16,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716113 2024-11-22T23:55:16,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716113 2024-11-22T23:55:16,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,120 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716113) roll requested 2024-11-22T23:55:16,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716120 2024-11-22T23:55:16,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716120 2024-11-22T23:55:16,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716120) roll requested 2024-11-22T23:55:16,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716126 2024-11-22T23:55:16,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716126 2024-11-22T23:55:16,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716126) roll requested 2024-11-22T23:55:16,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716133 2024-11-22T23:55:16,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716133 2024-11-22T23:55:16,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716133) roll requested 2024-11-22T23:55:16,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716141 2024-11-22T23:55:16,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716141 2024-11-22T23:55:16,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716141) roll requested 2024-11-22T23:55:16,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716149 2024-11-22T23:55:16,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716149 2024-11-22T23:55:16,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716149) roll requested 2024-11-22T23:55:16,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716155 2024-11-22T23:55:16,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716155 2024-11-22T23:55:16,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716155) roll requested 2024-11-22T23:55:16,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716162 2024-11-22T23:55:16,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716162 2024-11-22T23:55:16,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716162) roll requested 2024-11-22T23:55:16,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716169 2024-11-22T23:55:16,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716169 2024-11-22T23:55:16,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716169) roll requested 2024-11-22T23:55:16,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716177 2024-11-22T23:55:16,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716177 2024-11-22T23:55:16,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716177) roll requested 2024-11-22T23:55:16,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716184 2024-11-22T23:55:16,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716184 2024-11-22T23:55:16,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716184) roll requested 2024-11-22T23:55:16,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716190 2024-11-22T23:55:16,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716190 2024-11-22T23:55:16,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716190) roll requested 2024-11-22T23:55:16,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716197 2024-11-22T23:55:16,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716197 2024-11-22T23:55:16,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716197) roll requested 2024-11-22T23:55:16,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716204 2024-11-22T23:55:16,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716204 2024-11-22T23:55:16,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716204) roll requested 2024-11-22T23:55:16,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716211 2024-11-22T23:55:16,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716211 2024-11-22T23:55:16,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716211) roll requested 2024-11-22T23:55:16,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716218 2024-11-22T23:55:16,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716218 2024-11-22T23:55:16,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716218) roll requested 2024-11-22T23:55:16,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716225 2024-11-22T23:55:16,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716225 2024-11-22T23:55:16,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716225) roll requested 2024-11-22T23:55:16,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716232 2024-11-22T23:55:16,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716232 2024-11-22T23:55:16,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716232) roll requested 2024-11-22T23:55:16,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716239 2024-11-22T23:55:16,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716239 2024-11-22T23:55:16,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716239) roll requested 2024-11-22T23:55:16,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716246 2024-11-22T23:55:16,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716246 2024-11-22T23:55:16,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716246) roll requested 2024-11-22T23:55:16,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716252 2024-11-22T23:55:16,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716252 2024-11-22T23:55:16,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716252) roll requested 2024-11-22T23:55:16,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716259 2024-11-22T23:55:16,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716259 2024-11-22T23:55:16,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716259) roll requested 2024-11-22T23:55:16,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716266 2024-11-22T23:55:16,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716266 2024-11-22T23:55:16,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716266) roll requested 2024-11-22T23:55:16,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716273 2024-11-22T23:55:16,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716273 2024-11-22T23:55:16,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716273) roll requested 2024-11-22T23:55:16,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716280 2024-11-22T23:55:16,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716280 2024-11-22T23:55:16,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716280) roll requested 2024-11-22T23:55:16,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716287 2024-11-22T23:55:16,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716287 2024-11-22T23:55:16,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716287) roll requested 2024-11-22T23:55:16,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716293 2024-11-22T23:55:16,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716293 2024-11-22T23:55:16,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716293) roll requested 2024-11-22T23:55:16,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716300 2024-11-22T23:55:16,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716300 2024-11-22T23:55:16,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716300) roll requested 2024-11-22T23:55:16,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716306 2024-11-22T23:55:16,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716306 2024-11-22T23:55:16,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716306) roll requested 2024-11-22T23:55:16,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716313 2024-11-22T23:55:16,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716313 2024-11-22T23:55:16,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716313) roll requested 2024-11-22T23:55:16,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716320 2024-11-22T23:55:16,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716320 2024-11-22T23:55:16,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716320) roll requested 2024-11-22T23:55:16,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716327 2024-11-22T23:55:16,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716327 2024-11-22T23:55:16,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716327) roll requested 2024-11-22T23:55:16,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716334 2024-11-22T23:55:16,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716334 2024-11-22T23:55:16,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716334) roll requested 2024-11-22T23:55:16,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716342 2024-11-22T23:55:16,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716342 2024-11-22T23:55:16,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716342) roll requested 2024-11-22T23:55:16,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716349 2024-11-22T23:55:16,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716349 2024-11-22T23:55:16,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716349) roll requested 2024-11-22T23:55:16,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716356 2024-11-22T23:55:16,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716356 2024-11-22T23:55:16,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716356) roll requested 2024-11-22T23:55:16,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716363 2024-11-22T23:55:16,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716363 2024-11-22T23:55:16,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716363) roll requested 2024-11-22T23:55:16,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716370 2024-11-22T23:55:16,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716370 2024-11-22T23:55:16,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716370) roll requested 2024-11-22T23:55:16,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716377 2024-11-22T23:55:16,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716377 2024-11-22T23:55:16,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716377) roll requested 2024-11-22T23:55:16,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716384 2024-11-22T23:55:16,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716384 2024-11-22T23:55:16,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716384) roll requested 2024-11-22T23:55:16,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716390 2024-11-22T23:55:16,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716390 2024-11-22T23:55:16,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716390) roll requested 2024-11-22T23:55:16,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716397 2024-11-22T23:55:16,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716397 2024-11-22T23:55:16,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716397) roll requested 2024-11-22T23:55:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716404 2024-11-22T23:55:16,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716404 2024-11-22T23:55:16,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716404) roll requested 2024-11-22T23:55:16,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716410 2024-11-22T23:55:16,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716410 2024-11-22T23:55:16,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716410) roll requested 2024-11-22T23:55:16,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716417 2024-11-22T23:55:16,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716417 2024-11-22T23:55:16,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716417) roll requested 2024-11-22T23:55:16,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716424 2024-11-22T23:55:16,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716424 2024-11-22T23:55:16,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716424) roll requested 2024-11-22T23:55:16,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716431 2024-11-22T23:55:16,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716431 2024-11-22T23:55:16,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716431) roll requested 2024-11-22T23:55:16,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716438 2024-11-22T23:55:16,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716438 2024-11-22T23:55:16,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716438) roll requested 2024-11-22T23:55:16,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716448 2024-11-22T23:55:16,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716448 2024-11-22T23:55:16,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716448) roll requested 2024-11-22T23:55:16,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716455 2024-11-22T23:55:16,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716455 2024-11-22T23:55:16,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716455) roll requested 2024-11-22T23:55:16,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716462 2024-11-22T23:55:16,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716462 2024-11-22T23:55:16,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716462) roll requested 2024-11-22T23:55:16,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716468 2024-11-22T23:55:16,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716468 2024-11-22T23:55:16,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716468) roll requested 2024-11-22T23:55:16,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716476 2024-11-22T23:55:16,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716476 2024-11-22T23:55:16,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716476) roll requested 2024-11-22T23:55:16,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716483 2024-11-22T23:55:16,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716483 2024-11-22T23:55:16,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716483) roll requested 2024-11-22T23:55:16,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716489 2024-11-22T23:55:16,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716489 2024-11-22T23:55:16,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716489) roll requested 2024-11-22T23:55:16,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716497 2024-11-22T23:55:16,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716497 2024-11-22T23:55:16,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716497) roll requested 2024-11-22T23:55:16,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716504 2024-11-22T23:55:16,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716504 2024-11-22T23:55:16,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716504) roll requested 2024-11-22T23:55:16,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716511 2024-11-22T23:55:16,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716511 2024-11-22T23:55:16,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716511) roll requested 2024-11-22T23:55:16,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716518 2024-11-22T23:55:16,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716518 2024-11-22T23:55:16,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716518) roll requested 2024-11-22T23:55:16,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716525 2024-11-22T23:55:16,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716525 2024-11-22T23:55:16,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716525) roll requested 2024-11-22T23:55:16,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716532 2024-11-22T23:55:16,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716532 2024-11-22T23:55:16,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716532) roll requested 2024-11-22T23:55:16,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716539 2024-11-22T23:55:16,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716539 2024-11-22T23:55:16,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716539) roll requested 2024-11-22T23:55:16,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716546 2024-11-22T23:55:16,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716546 2024-11-22T23:55:16,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,553 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716546) roll requested 2024-11-22T23:55:16,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716555 2024-11-22T23:55:16,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716555 2024-11-22T23:55:16,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716555) roll requested 2024-11-22T23:55:16,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716562 2024-11-22T23:55:16,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716562 2024-11-22T23:55:16,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716562) roll requested 2024-11-22T23:55:16,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716568 2024-11-22T23:55:16,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716568 2024-11-22T23:55:16,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716568) roll requested 2024-11-22T23:55:16,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716575 2024-11-22T23:55:16,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716575 2024-11-22T23:55:16,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716575) roll requested 2024-11-22T23:55:16,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716582 2024-11-22T23:55:16,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716582 2024-11-22T23:55:16,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716582) roll requested 2024-11-22T23:55:16,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716589 2024-11-22T23:55:16,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716589 2024-11-22T23:55:16,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716589) roll requested 2024-11-22T23:55:16,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716596 2024-11-22T23:55:16,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716596 2024-11-22T23:55:16,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716596) roll requested 2024-11-22T23:55:16,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716603 2024-11-22T23:55:16,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716603 2024-11-22T23:55:16,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716603) roll requested 2024-11-22T23:55:16,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716610 2024-11-22T23:55:16,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716610 2024-11-22T23:55:16,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716610) roll requested 2024-11-22T23:55:16,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716617 2024-11-22T23:55:16,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716617 2024-11-22T23:55:16,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716617) roll requested 2024-11-22T23:55:16,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716624 2024-11-22T23:55:16,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716624 2024-11-22T23:55:16,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,631 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716624) roll requested 2024-11-22T23:55:16,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716632 2024-11-22T23:55:16,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716632 2024-11-22T23:55:16,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716632) roll requested 2024-11-22T23:55:16,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716639 2024-11-22T23:55:16,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716639 2024-11-22T23:55:16,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716639) roll requested 2024-11-22T23:55:16,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716646 2024-11-22T23:55:16,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716646 2024-11-22T23:55:16,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716646) roll requested 2024-11-22T23:55:16,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716653 2024-11-22T23:55:16,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716653 2024-11-22T23:55:16,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716653) roll requested 2024-11-22T23:55:16,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716660 2024-11-22T23:55:16,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716660 2024-11-22T23:55:16,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716660) roll requested 2024-11-22T23:55:16,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716667 2024-11-22T23:55:16,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716667 2024-11-22T23:55:16,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716667) roll requested 2024-11-22T23:55:16,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716674 2024-11-22T23:55:16,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716674 2024-11-22T23:55:16,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716674) roll requested 2024-11-22T23:55:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716681 2024-11-22T23:55:16,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716681 2024-11-22T23:55:16,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716681) roll requested 2024-11-22T23:55:16,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716688 2024-11-22T23:55:16,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716688 2024-11-22T23:55:16,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716688) roll requested 2024-11-22T23:55:16,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716695 2024-11-22T23:55:16,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716695 2024-11-22T23:55:16,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716695) roll requested 2024-11-22T23:55:16,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716702 2024-11-22T23:55:16,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716702 2024-11-22T23:55:16,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716702) roll requested 2024-11-22T23:55:16,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716709 2024-11-22T23:55:16,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716709 2024-11-22T23:55:16,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716709) roll requested 2024-11-22T23:55:16,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716716 2024-11-22T23:55:16,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716716 2024-11-22T23:55:16,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716716) roll requested 2024-11-22T23:55:16,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716724 2024-11-22T23:55:16,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716724 2024-11-22T23:55:16,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716724) roll requested 2024-11-22T23:55:16,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716731 2024-11-22T23:55:16,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716731 2024-11-22T23:55:16,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716731) roll requested 2024-11-22T23:55:16,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716739 2024-11-22T23:55:16,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716739 2024-11-22T23:55:16,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716739) roll requested 2024-11-22T23:55:16,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716745 2024-11-22T23:55:16,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716745 2024-11-22T23:55:16,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716745) roll requested 2024-11-22T23:55:16,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716753 2024-11-22T23:55:16,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716753 2024-11-22T23:55:16,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716753) roll requested 2024-11-22T23:55:16,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716760 2024-11-22T23:55:16,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716760 2024-11-22T23:55:16,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716760) roll requested 2024-11-22T23:55:16,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716767 2024-11-22T23:55:16,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716767 2024-11-22T23:55:16,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716767) roll requested 2024-11-22T23:55:16,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716774 2024-11-22T23:55:16,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716774 2024-11-22T23:55:16,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716774) roll requested 2024-11-22T23:55:16,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716782 2024-11-22T23:55:16,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716782 2024-11-22T23:55:16,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716782) roll requested 2024-11-22T23:55:16,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716788 2024-11-22T23:55:16,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716788 2024-11-22T23:55:16,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716788) roll requested 2024-11-22T23:55:16,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716795 2024-11-22T23:55:16,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716795 2024-11-22T23:55:16,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716795) roll requested 2024-11-22T23:55:16,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716802 2024-11-22T23:55:16,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716802 2024-11-22T23:55:16,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,809 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716802) roll requested 2024-11-22T23:55:16,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716809 2024-11-22T23:55:16,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716809 2024-11-22T23:55:16,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716809) roll requested 2024-11-22T23:55:16,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716816 2024-11-22T23:55:16,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716816 2024-11-22T23:55:16,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716816) roll requested 2024-11-22T23:55:16,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716823 2024-11-22T23:55:16,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716823 2024-11-22T23:55:16,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716823) roll requested 2024-11-22T23:55:16,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716830 2024-11-22T23:55:16,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716830 2024-11-22T23:55:16,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716830) roll requested 2024-11-22T23:55:16,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716837 2024-11-22T23:55:16,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716837 2024-11-22T23:55:16,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716837) roll requested 2024-11-22T23:55:16,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716844 2024-11-22T23:55:16,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716844 2024-11-22T23:55:16,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716844) roll requested 2024-11-22T23:55:16,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716851 2024-11-22T23:55:16,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716851 2024-11-22T23:55:16,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716851) roll requested 2024-11-22T23:55:16,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716858 2024-11-22T23:55:16,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716858 2024-11-22T23:55:16,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716858) roll requested 2024-11-22T23:55:16,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716865 2024-11-22T23:55:16,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716865 2024-11-22T23:55:16,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716865) roll requested 2024-11-22T23:55:16,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716872 2024-11-22T23:55:16,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716872 2024-11-22T23:55:16,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716872) roll requested 2024-11-22T23:55:16,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716879 2024-11-22T23:55:16,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716879 2024-11-22T23:55:16,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716879) roll requested 2024-11-22T23:55:16,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716886 2024-11-22T23:55:16,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716886 2024-11-22T23:55:16,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716886) roll requested 2024-11-22T23:55:16,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716893 2024-11-22T23:55:16,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716893 2024-11-22T23:55:16,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716893) roll requested 2024-11-22T23:55:16,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716904 2024-11-22T23:55:16,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716904 2024-11-22T23:55:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716904) roll requested 2024-11-22T23:55:16,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716911 2024-11-22T23:55:16,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716911 2024-11-22T23:55:16,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716911) roll requested 2024-11-22T23:55:16,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716918 2024-11-22T23:55:16,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716918 2024-11-22T23:55:16,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716918) roll requested 2024-11-22T23:55:16,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716925 2024-11-22T23:55:16,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716925 2024-11-22T23:55:16,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716925) roll requested 2024-11-22T23:55:16,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716932 2024-11-22T23:55:16,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716932 2024-11-22T23:55:16,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716932) roll requested 2024-11-22T23:55:16,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716939 2024-11-22T23:55:16,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716939 2024-11-22T23:55:16,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716939) roll requested 2024-11-22T23:55:16,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716946 2024-11-22T23:55:16,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716946 2024-11-22T23:55:16,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716946) roll requested 2024-11-22T23:55:16,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716952 2024-11-22T23:55:16,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716952 2024-11-22T23:55:16,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716952) roll requested 2024-11-22T23:55:16,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716959 2024-11-22T23:55:16,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716959 2024-11-22T23:55:16,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716959) roll requested 2024-11-22T23:55:16,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716970 2024-11-22T23:55:16,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716970 2024-11-22T23:55:16,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716970) roll requested 2024-11-22T23:55:16,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716977 2024-11-22T23:55:16,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716977 2024-11-22T23:55:16,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716977) roll requested 2024-11-22T23:55:16,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716984 2024-11-22T23:55:16,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716984 2024-11-22T23:55:16,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716984) roll requested 2024-11-22T23:55:16,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716990 2024-11-22T23:55:16,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:16,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716990 2024-11-22T23:55:16,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:16,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:16,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:16,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:16,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716990) roll requested 2024-11-22T23:55:16,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319716997 2024-11-22T23:55:17,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716997 2024-11-22T23:55:17,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319716997) roll requested 2024-11-22T23:55:17,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717004 2024-11-22T23:55:17,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319716997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717004 2024-11-22T23:55:17,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717004) roll requested 2024-11-22T23:55:17,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717012 2024-11-22T23:55:17,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717012 2024-11-22T23:55:17,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717012) roll requested 2024-11-22T23:55:17,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717019 2024-11-22T23:55:17,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717019 2024-11-22T23:55:17,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717019) roll requested 2024-11-22T23:55:17,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717026 2024-11-22T23:55:17,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717026 2024-11-22T23:55:17,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717026) roll requested 2024-11-22T23:55:17,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717033 2024-11-22T23:55:17,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717033 2024-11-22T23:55:17,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717033) roll requested 2024-11-22T23:55:17,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717041 2024-11-22T23:55:17,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717041 2024-11-22T23:55:17,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717041) roll requested 2024-11-22T23:55:17,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717048 2024-11-22T23:55:17,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717048 2024-11-22T23:55:17,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717048) roll requested 2024-11-22T23:55:17,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717055 2024-11-22T23:55:17,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717055 2024-11-22T23:55:17,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717055) roll requested 2024-11-22T23:55:17,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717062 2024-11-22T23:55:17,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717062 2024-11-22T23:55:17,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717062) roll requested 2024-11-22T23:55:17,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717070 2024-11-22T23:55:17,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717070 2024-11-22T23:55:17,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717070) roll requested 2024-11-22T23:55:17,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717077 2024-11-22T23:55:17,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717077 2024-11-22T23:55:17,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717077) roll requested 2024-11-22T23:55:17,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717084 2024-11-22T23:55:17,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717084 2024-11-22T23:55:17,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717084) roll requested 2024-11-22T23:55:17,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717091 2024-11-22T23:55:17,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717091 2024-11-22T23:55:17,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717091) roll requested 2024-11-22T23:55:17,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717097 2024-11-22T23:55:17,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717097 2024-11-22T23:55:17,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717097) roll requested 2024-11-22T23:55:17,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717104 2024-11-22T23:55:17,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717104 2024-11-22T23:55:17,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717104) roll requested 2024-11-22T23:55:17,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717111 2024-11-22T23:55:17,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717111 2024-11-22T23:55:17,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717111) roll requested 2024-11-22T23:55:17,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717119 2024-11-22T23:55:17,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717119 2024-11-22T23:55:17,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717119) roll requested 2024-11-22T23:55:17,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717131 2024-11-22T23:55:17,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717131 2024-11-22T23:55:17,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717131) roll requested 2024-11-22T23:55:17,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717138 2024-11-22T23:55:17,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717138 2024-11-22T23:55:17,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717138) roll requested 2024-11-22T23:55:17,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717144 2024-11-22T23:55:17,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717144 2024-11-22T23:55:17,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717144) roll requested 2024-11-22T23:55:17,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717151 2024-11-22T23:55:17,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717151 2024-11-22T23:55:17,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,158 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717151) roll requested 2024-11-22T23:55:17,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717158 2024-11-22T23:55:17,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717158 2024-11-22T23:55:17,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717158) roll requested 2024-11-22T23:55:17,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717165 2024-11-22T23:55:17,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717165 2024-11-22T23:55:17,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717165) roll requested 2024-11-22T23:55:17,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717173 2024-11-22T23:55:17,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717173 2024-11-22T23:55:17,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717173) roll requested 2024-11-22T23:55:17,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717180 2024-11-22T23:55:17,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717180 2024-11-22T23:55:17,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717180) roll requested 2024-11-22T23:55:17,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717187 2024-11-22T23:55:17,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717187 2024-11-22T23:55:17,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717187) roll requested 2024-11-22T23:55:17,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717194 2024-11-22T23:55:17,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717194 2024-11-22T23:55:17,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717194) roll requested 2024-11-22T23:55:17,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717200 2024-11-22T23:55:17,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717200 2024-11-22T23:55:17,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717200) roll requested 2024-11-22T23:55:17,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717207 2024-11-22T23:55:17,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717207 2024-11-22T23:55:17,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717207) roll requested 2024-11-22T23:55:17,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717214 2024-11-22T23:55:17,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717214 2024-11-22T23:55:17,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717214) roll requested 2024-11-22T23:55:17,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717221 2024-11-22T23:55:17,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717221 2024-11-22T23:55:17,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717221) roll requested 2024-11-22T23:55:17,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717230 2024-11-22T23:55:17,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717230 2024-11-22T23:55:17,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717230) roll requested 2024-11-22T23:55:17,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717237 2024-11-22T23:55:17,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717237 2024-11-22T23:55:17,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717237) roll requested 2024-11-22T23:55:17,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717244 2024-11-22T23:55:17,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717244 2024-11-22T23:55:17,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717244) roll requested 2024-11-22T23:55:17,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717251 2024-11-22T23:55:17,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717251 2024-11-22T23:55:17,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717251) roll requested 2024-11-22T23:55:17,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717258 2024-11-22T23:55:17,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717258 2024-11-22T23:55:17,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717258) roll requested 2024-11-22T23:55:17,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717265 2024-11-22T23:55:17,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717265 2024-11-22T23:55:17,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717265) roll requested 2024-11-22T23:55:17,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717272 2024-11-22T23:55:17,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717272 2024-11-22T23:55:17,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717272) roll requested 2024-11-22T23:55:17,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717279 2024-11-22T23:55:17,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717279 2024-11-22T23:55:17,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717279) roll requested 2024-11-22T23:55:17,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717286 2024-11-22T23:55:17,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717286 2024-11-22T23:55:17,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717286) roll requested 2024-11-22T23:55:17,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717293 2024-11-22T23:55:17,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717293 2024-11-22T23:55:17,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717293) roll requested 2024-11-22T23:55:17,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717300 2024-11-22T23:55:17,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717300 2024-11-22T23:55:17,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717300) roll requested 2024-11-22T23:55:17,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717307 2024-11-22T23:55:17,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717307 2024-11-22T23:55:17,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717307) roll requested 2024-11-22T23:55:17,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717315 2024-11-22T23:55:17,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717315 2024-11-22T23:55:17,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717315) roll requested 2024-11-22T23:55:17,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717323 2024-11-22T23:55:17,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717323 2024-11-22T23:55:17,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717323) roll requested 2024-11-22T23:55:17,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717330 2024-11-22T23:55:17,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717330 2024-11-22T23:55:17,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717330) roll requested 2024-11-22T23:55:17,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717336 2024-11-22T23:55:17,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717336 2024-11-22T23:55:17,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717336) roll requested 2024-11-22T23:55:17,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717343 2024-11-22T23:55:17,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717343 2024-11-22T23:55:17,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717343) roll requested 2024-11-22T23:55:17,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717350 2024-11-22T23:55:17,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717350 2024-11-22T23:55:17,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717350) roll requested 2024-11-22T23:55:17,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717357 2024-11-22T23:55:17,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717357 2024-11-22T23:55:17,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717357) roll requested 2024-11-22T23:55:17,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717364 2024-11-22T23:55:17,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717364 2024-11-22T23:55:17,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717364) roll requested 2024-11-22T23:55:17,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717371 2024-11-22T23:55:17,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717371 2024-11-22T23:55:17,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717371) roll requested 2024-11-22T23:55:17,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717378 2024-11-22T23:55:17,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717378 2024-11-22T23:55:17,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717378) roll requested 2024-11-22T23:55:17,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717385 2024-11-22T23:55:17,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717385 2024-11-22T23:55:17,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717385) roll requested 2024-11-22T23:55:17,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717392 2024-11-22T23:55:17,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717392 2024-11-22T23:55:17,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717392) roll requested 2024-11-22T23:55:17,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717398 2024-11-22T23:55:17,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717398 2024-11-22T23:55:17,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717398) roll requested 2024-11-22T23:55:17,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717405 2024-11-22T23:55:17,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717405 2024-11-22T23:55:17,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717405) roll requested 2024-11-22T23:55:17,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717411 2024-11-22T23:55:17,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717411 2024-11-22T23:55:17,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717411) roll requested 2024-11-22T23:55:17,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717418 2024-11-22T23:55:17,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717418 2024-11-22T23:55:17,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,429 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717418) roll requested 2024-11-22T23:55:17,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717429 2024-11-22T23:55:17,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717429 2024-11-22T23:55:17,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717429) roll requested 2024-11-22T23:55:17,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717435 2024-11-22T23:55:17,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717435 2024-11-22T23:55:17,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717435) roll requested 2024-11-22T23:55:17,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717443 2024-11-22T23:55:17,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717443 2024-11-22T23:55:17,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717443) roll requested 2024-11-22T23:55:17,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717450 2024-11-22T23:55:17,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717450 2024-11-22T23:55:17,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717450) roll requested 2024-11-22T23:55:17,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717457 2024-11-22T23:55:17,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717457 2024-11-22T23:55:17,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717457) roll requested 2024-11-22T23:55:17,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717464 2024-11-22T23:55:17,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717464 2024-11-22T23:55:17,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717464) roll requested 2024-11-22T23:55:17,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717471 2024-11-22T23:55:17,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717471 2024-11-22T23:55:17,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717471) roll requested 2024-11-22T23:55:17,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717478 2024-11-22T23:55:17,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717478 2024-11-22T23:55:17,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717478) roll requested 2024-11-22T23:55:17,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717485 2024-11-22T23:55:17,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717485 2024-11-22T23:55:17,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717485) roll requested 2024-11-22T23:55:17,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717492 2024-11-22T23:55:17,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717492 2024-11-22T23:55:17,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717492) roll requested 2024-11-22T23:55:17,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717498 2024-11-22T23:55:17,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717498 2024-11-22T23:55:17,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717498) roll requested 2024-11-22T23:55:17,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717505 2024-11-22T23:55:17,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717505 2024-11-22T23:55:17,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717505) roll requested 2024-11-22T23:55:17,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717512 2024-11-22T23:55:17,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717512 2024-11-22T23:55:17,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717512) roll requested 2024-11-22T23:55:17,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717519 2024-11-22T23:55:17,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717519 2024-11-22T23:55:17,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717519) roll requested 2024-11-22T23:55:17,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717526 2024-11-22T23:55:17,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717526 2024-11-22T23:55:17,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717526) roll requested 2024-11-22T23:55:17,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717533 2024-11-22T23:55:17,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717533 2024-11-22T23:55:17,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717533) roll requested 2024-11-22T23:55:17,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717541 2024-11-22T23:55:17,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717541 2024-11-22T23:55:17,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717541) roll requested 2024-11-22T23:55:17,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717548 2024-11-22T23:55:17,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717548 2024-11-22T23:55:17,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717548) roll requested 2024-11-22T23:55:17,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717555 2024-11-22T23:55:17,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717555 2024-11-22T23:55:17,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717555) roll requested 2024-11-22T23:55:17,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717562 2024-11-22T23:55:17,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717562 2024-11-22T23:55:17,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717562) roll requested 2024-11-22T23:55:17,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717569 2024-11-22T23:55:17,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717569 2024-11-22T23:55:17,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717569) roll requested 2024-11-22T23:55:17,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717576 2024-11-22T23:55:17,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717576 2024-11-22T23:55:17,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717576) roll requested 2024-11-22T23:55:17,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717583 2024-11-22T23:55:17,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717583 2024-11-22T23:55:17,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717583) roll requested 2024-11-22T23:55:17,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717589 2024-11-22T23:55:17,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717589 2024-11-22T23:55:17,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717589) roll requested 2024-11-22T23:55:17,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717596 2024-11-22T23:55:17,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717596 2024-11-22T23:55:17,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717596) roll requested 2024-11-22T23:55:17,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717602 2024-11-22T23:55:17,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717602 2024-11-22T23:55:17,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717602) roll requested 2024-11-22T23:55:17,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717609 2024-11-22T23:55:17,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717609 2024-11-22T23:55:17,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717609) roll requested 2024-11-22T23:55:17,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717616 2024-11-22T23:55:17,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717616 2024-11-22T23:55:17,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717616) roll requested 2024-11-22T23:55:17,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717624 2024-11-22T23:55:17,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717624 2024-11-22T23:55:17,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717624) roll requested 2024-11-22T23:55:17,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717630 2024-11-22T23:55:17,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717630 2024-11-22T23:55:17,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717630) roll requested 2024-11-22T23:55:17,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717637 2024-11-22T23:55:17,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717637 2024-11-22T23:55:17,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717637) roll requested 2024-11-22T23:55:17,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717645 2024-11-22T23:55:17,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717645 2024-11-22T23:55:17,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717645) roll requested 2024-11-22T23:55:17,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717652 2024-11-22T23:55:17,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717652 2024-11-22T23:55:17,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717652) roll requested 2024-11-22T23:55:17,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717659 2024-11-22T23:55:17,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717659 2024-11-22T23:55:17,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717659) roll requested 2024-11-22T23:55:17,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717666 2024-11-22T23:55:17,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717666 2024-11-22T23:55:17,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717666) roll requested 2024-11-22T23:55:17,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717674 2024-11-22T23:55:17,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717674 2024-11-22T23:55:17,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717674) roll requested 2024-11-22T23:55:17,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717681 2024-11-22T23:55:17,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717681 2024-11-22T23:55:17,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717681) roll requested 2024-11-22T23:55:17,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717688 2024-11-22T23:55:17,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717688 2024-11-22T23:55:17,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717688) roll requested 2024-11-22T23:55:17,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717694 2024-11-22T23:55:17,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717694 2024-11-22T23:55:17,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717694) roll requested 2024-11-22T23:55:17,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717702 2024-11-22T23:55:17,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717702 2024-11-22T23:55:17,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717702) roll requested 2024-11-22T23:55:17,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717709 2024-11-22T23:55:17,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717709 2024-11-22T23:55:17,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717709) roll requested 2024-11-22T23:55:17,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717716 2024-11-22T23:55:17,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717716 2024-11-22T23:55:17,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717716) roll requested 2024-11-22T23:55:17,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717723 2024-11-22T23:55:17,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717723 2024-11-22T23:55:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717723) roll requested 2024-11-22T23:55:17,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717730 2024-11-22T23:55:17,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717730 2024-11-22T23:55:17,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717730) roll requested 2024-11-22T23:55:17,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717736 2024-11-22T23:55:17,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717736 2024-11-22T23:55:17,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717736) roll requested 2024-11-22T23:55:17,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717743 2024-11-22T23:55:17,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717743 2024-11-22T23:55:17,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717743) roll requested 2024-11-22T23:55:17,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717749 2024-11-22T23:55:17,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717749 2024-11-22T23:55:17,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717749) roll requested 2024-11-22T23:55:17,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717756 2024-11-22T23:55:17,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717756 2024-11-22T23:55:17,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717756) roll requested 2024-11-22T23:55:17,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717763 2024-11-22T23:55:17,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717763 2024-11-22T23:55:17,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717763) roll requested 2024-11-22T23:55:17,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717771 2024-11-22T23:55:17,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717771 2024-11-22T23:55:17,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717771) roll requested 2024-11-22T23:55:17,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717778 2024-11-22T23:55:17,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717778 2024-11-22T23:55:17,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717778) roll requested 2024-11-22T23:55:17,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717785 2024-11-22T23:55:17,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717785 2024-11-22T23:55:17,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717785) roll requested 2024-11-22T23:55:17,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717792 2024-11-22T23:55:17,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717792 2024-11-22T23:55:17,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717792) roll requested 2024-11-22T23:55:17,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717799 2024-11-22T23:55:17,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717799 2024-11-22T23:55:17,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717799) roll requested 2024-11-22T23:55:17,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717806 2024-11-22T23:55:17,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717806 2024-11-22T23:55:17,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717806) roll requested 2024-11-22T23:55:17,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717813 2024-11-22T23:55:17,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717813 2024-11-22T23:55:17,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717813) roll requested 2024-11-22T23:55:17,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717820 2024-11-22T23:55:17,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717820 2024-11-22T23:55:17,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717820) roll requested 2024-11-22T23:55:17,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717828 2024-11-22T23:55:17,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717828 2024-11-22T23:55:17,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717828) roll requested 2024-11-22T23:55:17,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717835 2024-11-22T23:55:17,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717835 2024-11-22T23:55:17,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717835) roll requested 2024-11-22T23:55:17,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717842 2024-11-22T23:55:17,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717842 2024-11-22T23:55:17,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717842) roll requested 2024-11-22T23:55:17,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717850 2024-11-22T23:55:17,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717850 2024-11-22T23:55:17,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717850) roll requested 2024-11-22T23:55:17,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717857 2024-11-22T23:55:17,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717857 2024-11-22T23:55:17,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717857) roll requested 2024-11-22T23:55:17,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717864 2024-11-22T23:55:17,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717864 2024-11-22T23:55:17,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717864) roll requested 2024-11-22T23:55:17,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717872 2024-11-22T23:55:17,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717872 2024-11-22T23:55:17,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717872) roll requested 2024-11-22T23:55:17,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717879 2024-11-22T23:55:17,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717879 2024-11-22T23:55:17,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717879) roll requested 2024-11-22T23:55:17,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717888 2024-11-22T23:55:17,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717888 2024-11-22T23:55:17,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717888) roll requested 2024-11-22T23:55:17,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717894 2024-11-22T23:55:17,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717894 2024-11-22T23:55:17,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717894) roll requested 2024-11-22T23:55:17,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717902 2024-11-22T23:55:17,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717902 2024-11-22T23:55:17,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717902) roll requested 2024-11-22T23:55:17,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717909 2024-11-22T23:55:17,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717909 2024-11-22T23:55:17,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717909) roll requested 2024-11-22T23:55:17,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717916 2024-11-22T23:55:17,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717916 2024-11-22T23:55:17,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717916) roll requested 2024-11-22T23:55:17,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717923 2024-11-22T23:55:17,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717923 2024-11-22T23:55:17,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717923) roll requested 2024-11-22T23:55:17,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717930 2024-11-22T23:55:17,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717930 2024-11-22T23:55:17,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717930) roll requested 2024-11-22T23:55:17,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717937 2024-11-22T23:55:17,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717937 2024-11-22T23:55:17,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717937) roll requested 2024-11-22T23:55:17,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717944 2024-11-22T23:55:17,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717944 2024-11-22T23:55:17,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717944) roll requested 2024-11-22T23:55:17,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717951 2024-11-22T23:55:17,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717951 2024-11-22T23:55:17,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717951) roll requested 2024-11-22T23:55:17,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717958 2024-11-22T23:55:17,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717958 2024-11-22T23:55:17,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717958) roll requested 2024-11-22T23:55:17,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717965 2024-11-22T23:55:17,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717965 2024-11-22T23:55:17,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717965) roll requested 2024-11-22T23:55:17,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717975 2024-11-22T23:55:17,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717975 2024-11-22T23:55:17,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717975) roll requested 2024-11-22T23:55:17,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717982 2024-11-22T23:55:17,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717982 2024-11-22T23:55:17,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717982) roll requested 2024-11-22T23:55:17,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717989 2024-11-22T23:55:17,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:17,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717989 2024-11-22T23:55:17,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:17,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:17,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:17,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:17,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717989) roll requested 2024-11-22T23:55:17,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319717996 2024-11-22T23:55:18,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717996 2024-11-22T23:55:18,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319717996) roll requested 2024-11-22T23:55:18,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718003 2024-11-22T23:55:18,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319717996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718003 2024-11-22T23:55:18,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718003) roll requested 2024-11-22T23:55:18,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718010 2024-11-22T23:55:18,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718010 2024-11-22T23:55:18,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718010) roll requested 2024-11-22T23:55:18,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718017 2024-11-22T23:55:18,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718017 2024-11-22T23:55:18,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718017) roll requested 2024-11-22T23:55:18,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718024 2024-11-22T23:55:18,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718024 2024-11-22T23:55:18,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718024) roll requested 2024-11-22T23:55:18,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718032 2024-11-22T23:55:18,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718032 2024-11-22T23:55:18,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718032) roll requested 2024-11-22T23:55:18,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718038 2024-11-22T23:55:18,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718038 2024-11-22T23:55:18,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718038) roll requested 2024-11-22T23:55:18,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718045 2024-11-22T23:55:18,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718045 2024-11-22T23:55:18,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718045) roll requested 2024-11-22T23:55:18,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718052 2024-11-22T23:55:18,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718052 2024-11-22T23:55:18,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718052) roll requested 2024-11-22T23:55:18,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718059 2024-11-22T23:55:18,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718059 2024-11-22T23:55:18,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718059) roll requested 2024-11-22T23:55:18,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718066 2024-11-22T23:55:18,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718066 2024-11-22T23:55:18,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718066) roll requested 2024-11-22T23:55:18,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718073 2024-11-22T23:55:18,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718073 2024-11-22T23:55:18,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718073) roll requested 2024-11-22T23:55:18,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718080 2024-11-22T23:55:18,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718080 2024-11-22T23:55:18,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718080) roll requested 2024-11-22T23:55:18,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718087 2024-11-22T23:55:18,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718087 2024-11-22T23:55:18,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718087) roll requested 2024-11-22T23:55:18,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718094 2024-11-22T23:55:18,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718094 2024-11-22T23:55:18,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718094) roll requested 2024-11-22T23:55:18,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718101 2024-11-22T23:55:18,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718101 2024-11-22T23:55:18,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718101) roll requested 2024-11-22T23:55:18,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718108 2024-11-22T23:55:18,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718108 2024-11-22T23:55:18,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718108) roll requested 2024-11-22T23:55:18,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718115 2024-11-22T23:55:18,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718115 2024-11-22T23:55:18,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718115) roll requested 2024-11-22T23:55:18,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718122 2024-11-22T23:55:18,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718122 2024-11-22T23:55:18,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718122) roll requested 2024-11-22T23:55:18,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718129 2024-11-22T23:55:18,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718129 2024-11-22T23:55:18,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718129) roll requested 2024-11-22T23:55:18,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718135 2024-11-22T23:55:18,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718135 2024-11-22T23:55:18,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,141 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718135) roll requested 2024-11-22T23:55:18,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718141 2024-11-22T23:55:18,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718141 2024-11-22T23:55:18,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718141) roll requested 2024-11-22T23:55:18,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718148 2024-11-22T23:55:18,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718148 2024-11-22T23:55:18,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718148) roll requested 2024-11-22T23:55:18,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718155 2024-11-22T23:55:18,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718155 2024-11-22T23:55:18,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718155) roll requested 2024-11-22T23:55:18,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718162 2024-11-22T23:55:18,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718162 2024-11-22T23:55:18,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718162) roll requested 2024-11-22T23:55:18,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718168 2024-11-22T23:55:18,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718168 2024-11-22T23:55:18,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718168) roll requested 2024-11-22T23:55:18,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718175 2024-11-22T23:55:18,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718175 2024-11-22T23:55:18,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718175) roll requested 2024-11-22T23:55:18,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718182 2024-11-22T23:55:18,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718182 2024-11-22T23:55:18,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718182) roll requested 2024-11-22T23:55:18,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718188 2024-11-22T23:55:18,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718188 2024-11-22T23:55:18,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718188) roll requested 2024-11-22T23:55:18,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718195 2024-11-22T23:55:18,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718195 2024-11-22T23:55:18,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718195) roll requested 2024-11-22T23:55:18,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718209 2024-11-22T23:55:18,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718209 2024-11-22T23:55:18,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718209) roll requested 2024-11-22T23:55:18,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718217 2024-11-22T23:55:18,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718217 2024-11-22T23:55:18,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718217) roll requested 2024-11-22T23:55:18,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718224 2024-11-22T23:55:18,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718224 2024-11-22T23:55:18,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718224) roll requested 2024-11-22T23:55:18,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718231 2024-11-22T23:55:18,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718231 2024-11-22T23:55:18,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718231) roll requested 2024-11-22T23:55:18,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718238 2024-11-22T23:55:18,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718238 2024-11-22T23:55:18,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718238) roll requested 2024-11-22T23:55:18,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718245 2024-11-22T23:55:18,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718245 2024-11-22T23:55:18,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718245) roll requested 2024-11-22T23:55:18,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718251 2024-11-22T23:55:18,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718251 2024-11-22T23:55:18,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718251) roll requested 2024-11-22T23:55:18,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718258 2024-11-22T23:55:18,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718258 2024-11-22T23:55:18,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718258) roll requested 2024-11-22T23:55:18,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718265 2024-11-22T23:55:18,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718265 2024-11-22T23:55:18,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718265) roll requested 2024-11-22T23:55:18,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718272 2024-11-22T23:55:18,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718272 2024-11-22T23:55:18,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718272) roll requested 2024-11-22T23:55:18,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718279 2024-11-22T23:55:18,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718279 2024-11-22T23:55:18,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718279) roll requested 2024-11-22T23:55:18,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718286 2024-11-22T23:55:18,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718286 2024-11-22T23:55:18,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718286) roll requested 2024-11-22T23:55:18,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718292 2024-11-22T23:55:18,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718292 2024-11-22T23:55:18,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718292) roll requested 2024-11-22T23:55:18,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718299 2024-11-22T23:55:18,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718299 2024-11-22T23:55:18,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718299) roll requested 2024-11-22T23:55:18,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718306 2024-11-22T23:55:18,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718306 2024-11-22T23:55:18,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718306) roll requested 2024-11-22T23:55:18,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718313 2024-11-22T23:55:18,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718313 2024-11-22T23:55:18,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718313) roll requested 2024-11-22T23:55:18,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718319 2024-11-22T23:55:18,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718319 2024-11-22T23:55:18,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718319) roll requested 2024-11-22T23:55:18,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718326 2024-11-22T23:55:18,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718326 2024-11-22T23:55:18,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718326) roll requested 2024-11-22T23:55:18,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718333 2024-11-22T23:55:18,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718333 2024-11-22T23:55:18,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718333) roll requested 2024-11-22T23:55:18,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718341 2024-11-22T23:55:18,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718341 2024-11-22T23:55:18,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718341) roll requested 2024-11-22T23:55:18,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718348 2024-11-22T23:55:18,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718348 2024-11-22T23:55:18,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718348) roll requested 2024-11-22T23:55:18,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718355 2024-11-22T23:55:18,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718355 2024-11-22T23:55:18,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718355) roll requested 2024-11-22T23:55:18,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718362 2024-11-22T23:55:18,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718362 2024-11-22T23:55:18,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718362) roll requested 2024-11-22T23:55:18,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718369 2024-11-22T23:55:18,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718369 2024-11-22T23:55:18,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718369) roll requested 2024-11-22T23:55:18,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718376 2024-11-22T23:55:18,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718376 2024-11-22T23:55:18,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718376) roll requested 2024-11-22T23:55:18,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718383 2024-11-22T23:55:18,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718383 2024-11-22T23:55:18,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718383) roll requested 2024-11-22T23:55:18,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718390 2024-11-22T23:55:18,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718390 2024-11-22T23:55:18,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718390) roll requested 2024-11-22T23:55:18,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718397 2024-11-22T23:55:18,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718397 2024-11-22T23:55:18,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718397) roll requested 2024-11-22T23:55:18,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718403 2024-11-22T23:55:18,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718403 2024-11-22T23:55:18,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718403) roll requested 2024-11-22T23:55:18,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718410 2024-11-22T23:55:18,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718410 2024-11-22T23:55:18,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718410) roll requested 2024-11-22T23:55:18,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718417 2024-11-22T23:55:18,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718417 2024-11-22T23:55:18,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718417) roll requested 2024-11-22T23:55:18,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718424 2024-11-22T23:55:18,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718424 2024-11-22T23:55:18,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718424) roll requested 2024-11-22T23:55:18,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718431 2024-11-22T23:55:18,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718431 2024-11-22T23:55:18,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718431) roll requested 2024-11-22T23:55:18,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718439 2024-11-22T23:55:18,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718439 2024-11-22T23:55:18,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718439) roll requested 2024-11-22T23:55:18,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718445 2024-11-22T23:55:18,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718445 2024-11-22T23:55:18,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718445) roll requested 2024-11-22T23:55:18,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718452 2024-11-22T23:55:18,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718452 2024-11-22T23:55:18,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718452) roll requested 2024-11-22T23:55:18,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718460 2024-11-22T23:55:18,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718460 2024-11-22T23:55:18,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718460) roll requested 2024-11-22T23:55:18,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718467 2024-11-22T23:55:18,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718467 2024-11-22T23:55:18,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718467) roll requested 2024-11-22T23:55:18,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718474 2024-11-22T23:55:18,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718474 2024-11-22T23:55:18,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718474) roll requested 2024-11-22T23:55:18,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718481 2024-11-22T23:55:18,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718481 2024-11-22T23:55:18,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718481) roll requested 2024-11-22T23:55:18,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718488 2024-11-22T23:55:18,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718488 2024-11-22T23:55:18,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718488) roll requested 2024-11-22T23:55:18,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718497 2024-11-22T23:55:18,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718497 2024-11-22T23:55:18,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718497) roll requested 2024-11-22T23:55:18,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718504 2024-11-22T23:55:18,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718504 2024-11-22T23:55:18,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718504) roll requested 2024-11-22T23:55:18,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718512 2024-11-22T23:55:18,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718512 2024-11-22T23:55:18,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718512) roll requested 2024-11-22T23:55:18,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718519 2024-11-22T23:55:18,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718519 2024-11-22T23:55:18,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718519) roll requested 2024-11-22T23:55:18,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718526 2024-11-22T23:55:18,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718526 2024-11-22T23:55:18,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718526) roll requested 2024-11-22T23:55:18,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718533 2024-11-22T23:55:18,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718533 2024-11-22T23:55:18,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718533) roll requested 2024-11-22T23:55:18,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718540 2024-11-22T23:55:18,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718540 2024-11-22T23:55:18,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718540) roll requested 2024-11-22T23:55:18,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718547 2024-11-22T23:55:18,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718547 2024-11-22T23:55:18,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718547) roll requested 2024-11-22T23:55:18,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718556 2024-11-22T23:55:18,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718556 2024-11-22T23:55:18,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718556) roll requested 2024-11-22T23:55:18,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718564 2024-11-22T23:55:18,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718564 2024-11-22T23:55:18,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718564) roll requested 2024-11-22T23:55:18,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718571 2024-11-22T23:55:18,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718571 2024-11-22T23:55:18,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718571) roll requested 2024-11-22T23:55:18,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718578 2024-11-22T23:55:18,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718578 2024-11-22T23:55:18,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718578) roll requested 2024-11-22T23:55:18,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718585 2024-11-22T23:55:18,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718585 2024-11-22T23:55:18,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718585) roll requested 2024-11-22T23:55:18,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718595 2024-11-22T23:55:18,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718595 2024-11-22T23:55:18,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718595) roll requested 2024-11-22T23:55:18,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718603 2024-11-22T23:55:18,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718603 2024-11-22T23:55:18,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718603) roll requested 2024-11-22T23:55:18,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718609 2024-11-22T23:55:18,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718609 2024-11-22T23:55:18,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718609) roll requested 2024-11-22T23:55:18,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718617 2024-11-22T23:55:18,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718617 2024-11-22T23:55:18,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718617) roll requested 2024-11-22T23:55:18,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718626 2024-11-22T23:55:18,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718626 2024-11-22T23:55:18,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718626) roll requested 2024-11-22T23:55:18,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718633 2024-11-22T23:55:18,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718633 2024-11-22T23:55:18,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718633) roll requested 2024-11-22T23:55:18,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718640 2024-11-22T23:55:18,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718640 2024-11-22T23:55:18,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718640) roll requested 2024-11-22T23:55:18,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718653 2024-11-22T23:55:18,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718653 2024-11-22T23:55:18,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718653) roll requested 2024-11-22T23:55:18,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718664 2024-11-22T23:55:18,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718664 2024-11-22T23:55:18,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718664) roll requested 2024-11-22T23:55:18,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718672 2024-11-22T23:55:18,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718672 2024-11-22T23:55:18,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,684 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718672) roll requested 2024-11-22T23:55:18,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718684 2024-11-22T23:55:18,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718684 2024-11-22T23:55:18,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718684) roll requested 2024-11-22T23:55:18,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718691 2024-11-22T23:55:18,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718691 2024-11-22T23:55:18,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718691) roll requested 2024-11-22T23:55:18,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718699 2024-11-22T23:55:18,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718699 2024-11-22T23:55:18,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718699) roll requested 2024-11-22T23:55:18,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718706 2024-11-22T23:55:18,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718706 2024-11-22T23:55:18,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718706) roll requested 2024-11-22T23:55:18,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718713 2024-11-22T23:55:18,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718713 2024-11-22T23:55:18,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718713) roll requested 2024-11-22T23:55:18,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718721 2024-11-22T23:55:18,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718721 2024-11-22T23:55:18,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718721) roll requested 2024-11-22T23:55:18,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718728 2024-11-22T23:55:18,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718728 2024-11-22T23:55:18,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718728) roll requested 2024-11-22T23:55:18,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718735 2024-11-22T23:55:18,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718735 2024-11-22T23:55:18,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718735) roll requested 2024-11-22T23:55:18,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718742 2024-11-22T23:55:18,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718742 2024-11-22T23:55:18,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718742) roll requested 2024-11-22T23:55:18,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718749 2024-11-22T23:55:18,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718749 2024-11-22T23:55:18,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718749) roll requested 2024-11-22T23:55:18,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718756 2024-11-22T23:55:18,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718756 2024-11-22T23:55:18,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,763 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,763 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718756) roll requested 2024-11-22T23:55:18,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718763 2024-11-22T23:55:18,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718763 2024-11-22T23:55:18,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718763) roll requested 2024-11-22T23:55:18,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718770 2024-11-22T23:55:18,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718770 2024-11-22T23:55:18,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718770) roll requested 2024-11-22T23:55:18,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718777 2024-11-22T23:55:18,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718777 2024-11-22T23:55:18,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718777) roll requested 2024-11-22T23:55:18,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718784 2024-11-22T23:55:18,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718784 2024-11-22T23:55:18,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718784) roll requested 2024-11-22T23:55:18,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718791 2024-11-22T23:55:18,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718791 2024-11-22T23:55:18,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718791) roll requested 2024-11-22T23:55:18,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718811 2024-11-22T23:55:18,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718811 2024-11-22T23:55:18,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718811) roll requested 2024-11-22T23:55:18,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718825 2024-11-22T23:55:18,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718825 2024-11-22T23:55:18,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718825) roll requested 2024-11-22T23:55:18,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718833 2024-11-22T23:55:18,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718833 2024-11-22T23:55:18,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718833) roll requested 2024-11-22T23:55:18,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718840 2024-11-22T23:55:18,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718840 2024-11-22T23:55:18,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718840) roll requested 2024-11-22T23:55:18,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718849 2024-11-22T23:55:18,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718849 2024-11-22T23:55:18,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718849) roll requested 2024-11-22T23:55:18,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718857 2024-11-22T23:55:18,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718857 2024-11-22T23:55:18,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718857) roll requested 2024-11-22T23:55:18,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718864 2024-11-22T23:55:18,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718864 2024-11-22T23:55:18,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718864) roll requested 2024-11-22T23:55:18,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718871 2024-11-22T23:55:18,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718871 2024-11-22T23:55:18,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718871) roll requested 2024-11-22T23:55:18,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718878 2024-11-22T23:55:18,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718878 2024-11-22T23:55:18,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718878) roll requested 2024-11-22T23:55:18,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718886 2024-11-22T23:55:18,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718886 2024-11-22T23:55:18,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718886) roll requested 2024-11-22T23:55:18,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718892 2024-11-22T23:55:18,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718892 2024-11-22T23:55:18,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718892) roll requested 2024-11-22T23:55:18,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718900 2024-11-22T23:55:18,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718900 2024-11-22T23:55:18,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718900) roll requested 2024-11-22T23:55:18,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718907 2024-11-22T23:55:18,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718907 2024-11-22T23:55:18,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718907) roll requested 2024-11-22T23:55:18,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718914 2024-11-22T23:55:18,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718914 2024-11-22T23:55:18,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718914) roll requested 2024-11-22T23:55:18,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718922 2024-11-22T23:55:18,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718922 2024-11-22T23:55:18,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718922) roll requested 2024-11-22T23:55:18,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718930 2024-11-22T23:55:18,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718930 2024-11-22T23:55:18,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718930) roll requested 2024-11-22T23:55:18,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718937 2024-11-22T23:55:18,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718937 2024-11-22T23:55:18,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718937) roll requested 2024-11-22T23:55:18,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718943 2024-11-22T23:55:18,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718943 2024-11-22T23:55:18,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718943) roll requested 2024-11-22T23:55:18,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718950 2024-11-22T23:55:18,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718950 2024-11-22T23:55:18,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718950) roll requested 2024-11-22T23:55:18,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718958 2024-11-22T23:55:18,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718958 2024-11-22T23:55:18,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718958) roll requested 2024-11-22T23:55:18,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718965 2024-11-22T23:55:18,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718965 2024-11-22T23:55:18,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718965) roll requested 2024-11-22T23:55:18,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718971 2024-11-22T23:55:18,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718971 2024-11-22T23:55:18,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718971) roll requested 2024-11-22T23:55:18,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718978 2024-11-22T23:55:18,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718978 2024-11-22T23:55:18,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718978) roll requested 2024-11-22T23:55:18,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718985 2024-11-22T23:55:18,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718985 2024-11-22T23:55:18,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718985) roll requested 2024-11-22T23:55:18,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718991 2024-11-22T23:55:18,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:18,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718991 2024-11-22T23:55:18,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:18,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:18,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:18,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:18,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718991) roll requested 2024-11-22T23:55:18,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319718998 2024-11-22T23:55:19,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718998 2024-11-22T23:55:19,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319718998) roll requested 2024-11-22T23:55:19,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719004 2024-11-22T23:55:19,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319718998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719004 2024-11-22T23:55:19,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719004) roll requested 2024-11-22T23:55:19,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719011 2024-11-22T23:55:19,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719011 2024-11-22T23:55:19,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719011) roll requested 2024-11-22T23:55:19,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719018 2024-11-22T23:55:19,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719018 2024-11-22T23:55:19,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719018) roll requested 2024-11-22T23:55:19,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719026 2024-11-22T23:55:19,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719026 2024-11-22T23:55:19,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719026) roll requested 2024-11-22T23:55:19,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719034 2024-11-22T23:55:19,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719034 2024-11-22T23:55:19,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719034) roll requested 2024-11-22T23:55:19,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719041 2024-11-22T23:55:19,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719041 2024-11-22T23:55:19,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719041) roll requested 2024-11-22T23:55:19,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719048 2024-11-22T23:55:19,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719048 2024-11-22T23:55:19,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719048) roll requested 2024-11-22T23:55:19,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719059 2024-11-22T23:55:19,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719059 2024-11-22T23:55:19,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719059) roll requested 2024-11-22T23:55:19,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719069 2024-11-22T23:55:19,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719069 2024-11-22T23:55:19,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719069) roll requested 2024-11-22T23:55:19,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719076 2024-11-22T23:55:19,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719076 2024-11-22T23:55:19,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719076) roll requested 2024-11-22T23:55:19,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719084 2024-11-22T23:55:19,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719084 2024-11-22T23:55:19,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719084) roll requested 2024-11-22T23:55:19,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719090 2024-11-22T23:55:19,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719090 2024-11-22T23:55:19,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719090) roll requested 2024-11-22T23:55:19,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719097 2024-11-22T23:55:19,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719097 2024-11-22T23:55:19,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719097) roll requested 2024-11-22T23:55:19,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719104 2024-11-22T23:55:19,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719104 2024-11-22T23:55:19,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719104) roll requested 2024-11-22T23:55:19,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719126 2024-11-22T23:55:19,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719126 2024-11-22T23:55:19,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719126) roll requested 2024-11-22T23:55:19,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719133 2024-11-22T23:55:19,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719133 2024-11-22T23:55:19,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719133) roll requested 2024-11-22T23:55:19,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719140 2024-11-22T23:55:19,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719140 2024-11-22T23:55:19,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719140) roll requested 2024-11-22T23:55:19,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719147 2024-11-22T23:55:19,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719147 2024-11-22T23:55:19,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719147) roll requested 2024-11-22T23:55:19,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719154 2024-11-22T23:55:19,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719154 2024-11-22T23:55:19,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719154) roll requested 2024-11-22T23:55:19,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719165 2024-11-22T23:55:19,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719165 2024-11-22T23:55:19,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719165) roll requested 2024-11-22T23:55:19,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719173 2024-11-22T23:55:19,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719173 2024-11-22T23:55:19,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719173) roll requested 2024-11-22T23:55:19,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719181 2024-11-22T23:55:19,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719181 2024-11-22T23:55:19,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719181) roll requested 2024-11-22T23:55:19,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719188 2024-11-22T23:55:19,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719188 2024-11-22T23:55:19,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719188) roll requested 2024-11-22T23:55:19,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719195 2024-11-22T23:55:19,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719195 2024-11-22T23:55:19,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719195) roll requested 2024-11-22T23:55:19,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719202 2024-11-22T23:55:19,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719202 2024-11-22T23:55:19,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719202) roll requested 2024-11-22T23:55:19,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719210 2024-11-22T23:55:19,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719210 2024-11-22T23:55:19,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719210) roll requested 2024-11-22T23:55:19,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719217 2024-11-22T23:55:19,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719217 2024-11-22T23:55:19,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719217) roll requested 2024-11-22T23:55:19,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719225 2024-11-22T23:55:19,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719225 2024-11-22T23:55:19,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719225) roll requested 2024-11-22T23:55:19,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719232 2024-11-22T23:55:19,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719232 2024-11-22T23:55:19,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719232) roll requested 2024-11-22T23:55:19,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719239 2024-11-22T23:55:19,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719239 2024-11-22T23:55:19,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719239) roll requested 2024-11-22T23:55:19,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719246 2024-11-22T23:55:19,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719246 2024-11-22T23:55:19,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719246) roll requested 2024-11-22T23:55:19,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719253 2024-11-22T23:55:19,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719253 2024-11-22T23:55:19,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719253) roll requested 2024-11-22T23:55:19,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719260 2024-11-22T23:55:19,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719260 2024-11-22T23:55:19,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719260) roll requested 2024-11-22T23:55:19,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719267 2024-11-22T23:55:19,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719267 2024-11-22T23:55:19,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719267) roll requested 2024-11-22T23:55:19,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719275 2024-11-22T23:55:19,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719275 2024-11-22T23:55:19,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719275) roll requested 2024-11-22T23:55:19,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719282 2024-11-22T23:55:19,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719282 2024-11-22T23:55:19,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719282) roll requested 2024-11-22T23:55:19,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719289 2024-11-22T23:55:19,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719289 2024-11-22T23:55:19,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719289) roll requested 2024-11-22T23:55:19,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719299 2024-11-22T23:55:19,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719299 2024-11-22T23:55:19,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719299) roll requested 2024-11-22T23:55:19,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719306 2024-11-22T23:55:19,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719306 2024-11-22T23:55:19,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719306) roll requested 2024-11-22T23:55:19,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719313 2024-11-22T23:55:19,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719313 2024-11-22T23:55:19,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719313) roll requested 2024-11-22T23:55:19,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719322 2024-11-22T23:55:19,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719322 2024-11-22T23:55:19,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719322) roll requested 2024-11-22T23:55:19,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719329 2024-11-22T23:55:19,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719329 2024-11-22T23:55:19,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719329) roll requested 2024-11-22T23:55:19,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719336 2024-11-22T23:55:19,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719336 2024-11-22T23:55:19,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719336) roll requested 2024-11-22T23:55:19,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719343 2024-11-22T23:55:19,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719343 2024-11-22T23:55:19,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719343) roll requested 2024-11-22T23:55:19,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719350 2024-11-22T23:55:19,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719350 2024-11-22T23:55:19,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719350) roll requested 2024-11-22T23:55:19,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719358 2024-11-22T23:55:19,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719358 2024-11-22T23:55:19,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719358) roll requested 2024-11-22T23:55:19,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719365 2024-11-22T23:55:19,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719365 2024-11-22T23:55:19,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719365) roll requested 2024-11-22T23:55:19,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719372 2024-11-22T23:55:19,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719372 2024-11-22T23:55:19,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719372) roll requested 2024-11-22T23:55:19,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719378 2024-11-22T23:55:19,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719378 2024-11-22T23:55:19,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719378) roll requested 2024-11-22T23:55:19,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719385 2024-11-22T23:55:19,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719385 2024-11-22T23:55:19,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719385) roll requested 2024-11-22T23:55:19,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719392 2024-11-22T23:55:19,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719392 2024-11-22T23:55:19,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719392) roll requested 2024-11-22T23:55:19,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719399 2024-11-22T23:55:19,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719399 2024-11-22T23:55:19,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719399) roll requested 2024-11-22T23:55:19,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719405 2024-11-22T23:55:19,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719405 2024-11-22T23:55:19,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719405) roll requested 2024-11-22T23:55:19,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719413 2024-11-22T23:55:19,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719413 2024-11-22T23:55:19,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719413) roll requested 2024-11-22T23:55:19,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719420 2024-11-22T23:55:19,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719420 2024-11-22T23:55:19,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719420) roll requested 2024-11-22T23:55:19,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719427 2024-11-22T23:55:19,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719427 2024-11-22T23:55:19,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719427) roll requested 2024-11-22T23:55:19,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719434 2024-11-22T23:55:19,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719434 2024-11-22T23:55:19,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719434) roll requested 2024-11-22T23:55:19,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719441 2024-11-22T23:55:19,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719441 2024-11-22T23:55:19,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719441) roll requested 2024-11-22T23:55:19,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719448 2024-11-22T23:55:19,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719448 2024-11-22T23:55:19,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719448) roll requested 2024-11-22T23:55:19,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719455 2024-11-22T23:55:19,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719455 2024-11-22T23:55:19,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719455) roll requested 2024-11-22T23:55:19,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719462 2024-11-22T23:55:19,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719462 2024-11-22T23:55:19,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719462) roll requested 2024-11-22T23:55:19,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719470 2024-11-22T23:55:19,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719470 2024-11-22T23:55:19,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719470) roll requested 2024-11-22T23:55:19,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719477 2024-11-22T23:55:19,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719477 2024-11-22T23:55:19,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719477) roll requested 2024-11-22T23:55:19,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719484 2024-11-22T23:55:19,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719484 2024-11-22T23:55:19,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719484) roll requested 2024-11-22T23:55:19,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719491 2024-11-22T23:55:19,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719491 2024-11-22T23:55:19,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719491) roll requested 2024-11-22T23:55:19,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719498 2024-11-22T23:55:19,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719498 2024-11-22T23:55:19,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719498) roll requested 2024-11-22T23:55:19,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719504 2024-11-22T23:55:19,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719504 2024-11-22T23:55:19,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719504) roll requested 2024-11-22T23:55:19,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719511 2024-11-22T23:55:19,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719511 2024-11-22T23:55:19,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719511) roll requested 2024-11-22T23:55:19,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719518 2024-11-22T23:55:19,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719518 2024-11-22T23:55:19,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719518) roll requested 2024-11-22T23:55:19,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719525 2024-11-22T23:55:19,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719525 2024-11-22T23:55:19,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719525) roll requested 2024-11-22T23:55:19,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719534 2024-11-22T23:55:19,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719534 2024-11-22T23:55:19,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719534) roll requested 2024-11-22T23:55:19,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719541 2024-11-22T23:55:19,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719541 2024-11-22T23:55:19,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719541) roll requested 2024-11-22T23:55:19,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719549 2024-11-22T23:55:19,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719549 2024-11-22T23:55:19,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719549) roll requested 2024-11-22T23:55:19,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719556 2024-11-22T23:55:19,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719556 2024-11-22T23:55:19,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719556) roll requested 2024-11-22T23:55:19,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719563 2024-11-22T23:55:19,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719563 2024-11-22T23:55:19,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719563) roll requested 2024-11-22T23:55:19,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719569 2024-11-22T23:55:19,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719569 2024-11-22T23:55:19,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719569) roll requested 2024-11-22T23:55:19,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719576 2024-11-22T23:55:19,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719576 2024-11-22T23:55:19,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719576) roll requested 2024-11-22T23:55:19,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719593 2024-11-22T23:55:19,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719593 2024-11-22T23:55:19,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719593) roll requested 2024-11-22T23:55:19,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719602 2024-11-22T23:55:19,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719602 2024-11-22T23:55:19,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719602) roll requested 2024-11-22T23:55:19,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719613 2024-11-22T23:55:19,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719613 2024-11-22T23:55:19,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719613) roll requested 2024-11-22T23:55:19,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719621 2024-11-22T23:55:19,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719621 2024-11-22T23:55:19,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719621) roll requested 2024-11-22T23:55:19,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719637 2024-11-22T23:55:19,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719637 2024-11-22T23:55:19,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719637) roll requested 2024-11-22T23:55:19,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719660 2024-11-22T23:55:19,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719660 2024-11-22T23:55:19,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719660) roll requested 2024-11-22T23:55:19,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719669 2024-11-22T23:55:19,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719669 2024-11-22T23:55:19,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719669) roll requested 2024-11-22T23:55:19,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719677 2024-11-22T23:55:19,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719677 2024-11-22T23:55:19,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719677) roll requested 2024-11-22T23:55:19,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719689 2024-11-22T23:55:19,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719689 2024-11-22T23:55:19,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719689) roll requested 2024-11-22T23:55:19,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719697 2024-11-22T23:55:19,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719697 2024-11-22T23:55:19,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719697) roll requested 2024-11-22T23:55:19,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719705 2024-11-22T23:55:19,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719705 2024-11-22T23:55:19,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719705) roll requested 2024-11-22T23:55:19,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719712 2024-11-22T23:55:19,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719712 2024-11-22T23:55:19,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,719 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719712) roll requested 2024-11-22T23:55:19,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719719 2024-11-22T23:55:19,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719719 2024-11-22T23:55:19,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719719) roll requested 2024-11-22T23:55:19,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719725 2024-11-22T23:55:19,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719725 2024-11-22T23:55:19,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719725) roll requested 2024-11-22T23:55:19,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719733 2024-11-22T23:55:19,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719733 2024-11-22T23:55:19,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719733) roll requested 2024-11-22T23:55:19,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719741 2024-11-22T23:55:19,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719741 2024-11-22T23:55:19,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719741) roll requested 2024-11-22T23:55:19,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719748 2024-11-22T23:55:19,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719748 2024-11-22T23:55:19,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719748) roll requested 2024-11-22T23:55:19,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719755 2024-11-22T23:55:19,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719755 2024-11-22T23:55:19,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719755) roll requested 2024-11-22T23:55:19,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719762 2024-11-22T23:55:19,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719762 2024-11-22T23:55:19,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719762) roll requested 2024-11-22T23:55:19,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719770 2024-11-22T23:55:19,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719770 2024-11-22T23:55:19,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,777 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719770) roll requested 2024-11-22T23:55:19,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719777 2024-11-22T23:55:19,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719777 2024-11-22T23:55:19,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719777) roll requested 2024-11-22T23:55:19,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719784 2024-11-22T23:55:19,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719784 2024-11-22T23:55:19,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719784) roll requested 2024-11-22T23:55:19,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719791 2024-11-22T23:55:19,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719791 2024-11-22T23:55:19,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719791) roll requested 2024-11-22T23:55:19,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719798 2024-11-22T23:55:19,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719798 2024-11-22T23:55:19,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719798) roll requested 2024-11-22T23:55:19,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719805 2024-11-22T23:55:19,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719805 2024-11-22T23:55:19,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719805) roll requested 2024-11-22T23:55:19,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719817 2024-11-22T23:55:19,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719817 2024-11-22T23:55:19,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719817) roll requested 2024-11-22T23:55:19,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719824 2024-11-22T23:55:19,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719824 2024-11-22T23:55:19,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719824) roll requested 2024-11-22T23:55:19,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719831 2024-11-22T23:55:19,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719831 2024-11-22T23:55:19,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719831) roll requested 2024-11-22T23:55:19,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719838 2024-11-22T23:55:19,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719838 2024-11-22T23:55:19,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719838) roll requested 2024-11-22T23:55:19,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719845 2024-11-22T23:55:19,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719845 2024-11-22T23:55:19,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719845) roll requested 2024-11-22T23:55:19,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719852 2024-11-22T23:55:19,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719852 2024-11-22T23:55:19,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719852) roll requested 2024-11-22T23:55:19,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719858 2024-11-22T23:55:19,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719858 2024-11-22T23:55:19,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719858) roll requested 2024-11-22T23:55:19,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719865 2024-11-22T23:55:19,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719865 2024-11-22T23:55:19,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719865) roll requested 2024-11-22T23:55:19,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719872 2024-11-22T23:55:19,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719872 2024-11-22T23:55:19,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719872) roll requested 2024-11-22T23:55:19,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719879 2024-11-22T23:55:19,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719879 2024-11-22T23:55:19,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719879) roll requested 2024-11-22T23:55:19,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719886 2024-11-22T23:55:19,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719886 2024-11-22T23:55:19,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719886) roll requested 2024-11-22T23:55:19,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719893 2024-11-22T23:55:19,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719893 2024-11-22T23:55:19,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719893) roll requested 2024-11-22T23:55:19,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719900 2024-11-22T23:55:19,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719900 2024-11-22T23:55:19,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719900) roll requested 2024-11-22T23:55:19,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719907 2024-11-22T23:55:19,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719907 2024-11-22T23:55:19,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719907) roll requested 2024-11-22T23:55:19,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719914 2024-11-22T23:55:19,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719914 2024-11-22T23:55:19,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719914) roll requested 2024-11-22T23:55:19,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719920 2024-11-22T23:55:19,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719920 2024-11-22T23:55:19,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719920) roll requested 2024-11-22T23:55:19,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719927 2024-11-22T23:55:19,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719927 2024-11-22T23:55:19,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719927) roll requested 2024-11-22T23:55:19,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719934 2024-11-22T23:55:19,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719934 2024-11-22T23:55:19,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719934) roll requested 2024-11-22T23:55:19,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719941 2024-11-22T23:55:19,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719941 2024-11-22T23:55:19,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719941) roll requested 2024-11-22T23:55:19,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719949 2024-11-22T23:55:19,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719949 2024-11-22T23:55:19,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719949) roll requested 2024-11-22T23:55:19,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719956 2024-11-22T23:55:19,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719956 2024-11-22T23:55:19,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,963 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719956) roll requested 2024-11-22T23:55:19,963 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719963 2024-11-22T23:55:19,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719956 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719963 2024-11-22T23:55:19,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719963) roll requested 2024-11-22T23:55:19,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719971 2024-11-22T23:55:19,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719963 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719971 2024-11-22T23:55:19,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719971) roll requested 2024-11-22T23:55:19,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719978 2024-11-22T23:55:19,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719978 2024-11-22T23:55:19,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719978) roll requested 2024-11-22T23:55:19,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719985 2024-11-22T23:55:19,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719985 2024-11-22T23:55:19,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719985) roll requested 2024-11-22T23:55:19,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719991 2024-11-22T23:55:19,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:19,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719991 2024-11-22T23:55:19,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:19,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:19,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:19,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:19,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719991) roll requested 2024-11-22T23:55:19,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319719998 2024-11-22T23:55:20,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719998 2024-11-22T23:55:20,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319719998) roll requested 2024-11-22T23:55:20,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720005 2024-11-22T23:55:20,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319719998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720005 2024-11-22T23:55:20,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720005) roll requested 2024-11-22T23:55:20,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720012 2024-11-22T23:55:20,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720012 2024-11-22T23:55:20,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720012) roll requested 2024-11-22T23:55:20,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720021 2024-11-22T23:55:20,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720021 2024-11-22T23:55:20,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720021) roll requested 2024-11-22T23:55:20,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720027 2024-11-22T23:55:20,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720027 2024-11-22T23:55:20,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720027) roll requested 2024-11-22T23:55:20,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720034 2024-11-22T23:55:20,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720034 2024-11-22T23:55:20,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720034) roll requested 2024-11-22T23:55:20,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720041 2024-11-22T23:55:20,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720041 2024-11-22T23:55:20,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720041) roll requested 2024-11-22T23:55:20,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720048 2024-11-22T23:55:20,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720048 2024-11-22T23:55:20,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720048) roll requested 2024-11-22T23:55:20,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720055 2024-11-22T23:55:20,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720055 2024-11-22T23:55:20,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720055) roll requested 2024-11-22T23:55:20,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720062 2024-11-22T23:55:20,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720062 2024-11-22T23:55:20,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720062) roll requested 2024-11-22T23:55:20,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720069 2024-11-22T23:55:20,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720069 2024-11-22T23:55:20,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720069) roll requested 2024-11-22T23:55:20,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720076 2024-11-22T23:55:20,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720076 2024-11-22T23:55:20,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720076) roll requested 2024-11-22T23:55:20,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720083 2024-11-22T23:55:20,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720083 2024-11-22T23:55:20,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720083) roll requested 2024-11-22T23:55:20,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720090 2024-11-22T23:55:20,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720090 2024-11-22T23:55:20,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720090) roll requested 2024-11-22T23:55:20,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720097 2024-11-22T23:55:20,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720097 2024-11-22T23:55:20,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720097) roll requested 2024-11-22T23:55:20,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720104 2024-11-22T23:55:20,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720104 2024-11-22T23:55:20,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720104) roll requested 2024-11-22T23:55:20,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720111 2024-11-22T23:55:20,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720111 2024-11-22T23:55:20,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720111) roll requested 2024-11-22T23:55:20,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720117 2024-11-22T23:55:20,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720117 2024-11-22T23:55:20,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720117) roll requested 2024-11-22T23:55:20,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720124 2024-11-22T23:55:20,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720124 2024-11-22T23:55:20,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720124) roll requested 2024-11-22T23:55:20,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720131 2024-11-22T23:55:20,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720131 2024-11-22T23:55:20,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720131) roll requested 2024-11-22T23:55:20,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720138 2024-11-22T23:55:20,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720131 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720138 2024-11-22T23:55:20,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720138) roll requested 2024-11-22T23:55:20,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720145 2024-11-22T23:55:20,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720145 2024-11-22T23:55:20,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720145) roll requested 2024-11-22T23:55:20,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720151 2024-11-22T23:55:20,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720151 2024-11-22T23:55:20,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720151) roll requested 2024-11-22T23:55:20,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720159 2024-11-22T23:55:20,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720159 2024-11-22T23:55:20,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720159) roll requested 2024-11-22T23:55:20,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720166 2024-11-22T23:55:20,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720166 2024-11-22T23:55:20,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720166) roll requested 2024-11-22T23:55:20,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720173 2024-11-22T23:55:20,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720173 2024-11-22T23:55:20,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720173) roll requested 2024-11-22T23:55:20,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720180 2024-11-22T23:55:20,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720180 2024-11-22T23:55:20,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720180) roll requested 2024-11-22T23:55:20,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720187 2024-11-22T23:55:20,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720187 2024-11-22T23:55:20,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720187) roll requested 2024-11-22T23:55:20,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720194 2024-11-22T23:55:20,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720194 2024-11-22T23:55:20,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720194) roll requested 2024-11-22T23:55:20,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720201 2024-11-22T23:55:20,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720201 2024-11-22T23:55:20,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720201) roll requested 2024-11-22T23:55:20,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720208 2024-11-22T23:55:20,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720208 2024-11-22T23:55:20,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720208) roll requested 2024-11-22T23:55:20,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720215 2024-11-22T23:55:20,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720215 2024-11-22T23:55:20,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720215) roll requested 2024-11-22T23:55:20,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720222 2024-11-22T23:55:20,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720222 2024-11-22T23:55:20,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720222) roll requested 2024-11-22T23:55:20,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720229 2024-11-22T23:55:20,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720229 2024-11-22T23:55:20,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720229) roll requested 2024-11-22T23:55:20,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720237 2024-11-22T23:55:20,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720237 2024-11-22T23:55:20,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720237) roll requested 2024-11-22T23:55:20,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720244 2024-11-22T23:55:20,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720244 2024-11-22T23:55:20,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720244) roll requested 2024-11-22T23:55:20,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720252 2024-11-22T23:55:20,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720252 2024-11-22T23:55:20,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720252) roll requested 2024-11-22T23:55:20,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720259 2024-11-22T23:55:20,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720259 2024-11-22T23:55:20,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720259) roll requested 2024-11-22T23:55:20,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720266 2024-11-22T23:55:20,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720266 2024-11-22T23:55:20,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720266) roll requested 2024-11-22T23:55:20,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720273 2024-11-22T23:55:20,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720273 2024-11-22T23:55:20,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720273) roll requested 2024-11-22T23:55:20,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720280 2024-11-22T23:55:20,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720280 2024-11-22T23:55:20,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720280) roll requested 2024-11-22T23:55:20,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720287 2024-11-22T23:55:20,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720287 2024-11-22T23:55:20,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,294 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720287) roll requested 2024-11-22T23:55:20,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720294 2024-11-22T23:55:20,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720294 2024-11-22T23:55:20,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720294) roll requested 2024-11-22T23:55:20,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720301 2024-11-22T23:55:20,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720301 2024-11-22T23:55:20,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720301) roll requested 2024-11-22T23:55:20,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720307 2024-11-22T23:55:20,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720307 2024-11-22T23:55:20,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720307) roll requested 2024-11-22T23:55:20,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720314 2024-11-22T23:55:20,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720314 2024-11-22T23:55:20,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720314) roll requested 2024-11-22T23:55:20,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720322 2024-11-22T23:55:20,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720322 2024-11-22T23:55:20,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720322) roll requested 2024-11-22T23:55:20,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720328 2024-11-22T23:55:20,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720328 2024-11-22T23:55:20,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720328) roll requested 2024-11-22T23:55:20,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720335 2024-11-22T23:55:20,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720335 2024-11-22T23:55:20,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720335) roll requested 2024-11-22T23:55:20,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720342 2024-11-22T23:55:20,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720342 2024-11-22T23:55:20,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720342) roll requested 2024-11-22T23:55:20,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720349 2024-11-22T23:55:20,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720349 2024-11-22T23:55:20,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720349) roll requested 2024-11-22T23:55:20,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720356 2024-11-22T23:55:20,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720356 2024-11-22T23:55:20,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720356) roll requested 2024-11-22T23:55:20,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720362 2024-11-22T23:55:20,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720362 2024-11-22T23:55:20,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720362) roll requested 2024-11-22T23:55:20,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720369 2024-11-22T23:55:20,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720369 2024-11-22T23:55:20,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720369) roll requested 2024-11-22T23:55:20,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720376 2024-11-22T23:55:20,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720376 2024-11-22T23:55:20,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720376) roll requested 2024-11-22T23:55:20,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720383 2024-11-22T23:55:20,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720383 2024-11-22T23:55:20,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720383) roll requested 2024-11-22T23:55:20,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720390 2024-11-22T23:55:20,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720390 2024-11-22T23:55:20,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720390) roll requested 2024-11-22T23:55:20,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720397 2024-11-22T23:55:20,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720397 2024-11-22T23:55:20,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720397) roll requested 2024-11-22T23:55:20,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720404 2024-11-22T23:55:20,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720404 2024-11-22T23:55:20,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720404) roll requested 2024-11-22T23:55:20,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720411 2024-11-22T23:55:20,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,417 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,417 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720411 2024-11-22T23:55:20,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720411) roll requested 2024-11-22T23:55:20,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720418 2024-11-22T23:55:20,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720418 2024-11-22T23:55:20,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720418) roll requested 2024-11-22T23:55:20,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720428 2024-11-22T23:55:20,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720428 2024-11-22T23:55:20,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720428) roll requested 2024-11-22T23:55:20,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720435 2024-11-22T23:55:20,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720435 2024-11-22T23:55:20,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720435) roll requested 2024-11-22T23:55:20,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720442 2024-11-22T23:55:20,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720442 2024-11-22T23:55:20,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720442) roll requested 2024-11-22T23:55:20,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720449 2024-11-22T23:55:20,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720449 2024-11-22T23:55:20,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720449) roll requested 2024-11-22T23:55:20,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720455 2024-11-22T23:55:20,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720455 2024-11-22T23:55:20,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720455) roll requested 2024-11-22T23:55:20,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720462 2024-11-22T23:55:20,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720462 2024-11-22T23:55:20,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720462) roll requested 2024-11-22T23:55:20,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720469 2024-11-22T23:55:20,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720469 2024-11-22T23:55:20,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720469) roll requested 2024-11-22T23:55:20,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720478 2024-11-22T23:55:20,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720478 2024-11-22T23:55:20,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720478) roll requested 2024-11-22T23:55:20,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720485 2024-11-22T23:55:20,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720485 2024-11-22T23:55:20,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720485) roll requested 2024-11-22T23:55:20,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720492 2024-11-22T23:55:20,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720492 2024-11-22T23:55:20,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720492) roll requested 2024-11-22T23:55:20,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720499 2024-11-22T23:55:20,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720499 2024-11-22T23:55:20,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720499) roll requested 2024-11-22T23:55:20,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720506 2024-11-22T23:55:20,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720506 2024-11-22T23:55:20,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720506) roll requested 2024-11-22T23:55:20,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720512 2024-11-22T23:55:20,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720512 2024-11-22T23:55:20,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720512) roll requested 2024-11-22T23:55:20,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720521 2024-11-22T23:55:20,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720521 2024-11-22T23:55:20,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720521) roll requested 2024-11-22T23:55:20,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720528 2024-11-22T23:55:20,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720528 2024-11-22T23:55:20,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720528) roll requested 2024-11-22T23:55:20,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720535 2024-11-22T23:55:20,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720535 2024-11-22T23:55:20,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720535) roll requested 2024-11-22T23:55:20,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720542 2024-11-22T23:55:20,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720542 2024-11-22T23:55:20,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720542) roll requested 2024-11-22T23:55:20,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720549 2024-11-22T23:55:20,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720549 2024-11-22T23:55:20,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720549) roll requested 2024-11-22T23:55:20,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720556 2024-11-22T23:55:20,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720556 2024-11-22T23:55:20,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720556) roll requested 2024-11-22T23:55:20,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720563 2024-11-22T23:55:20,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720563 2024-11-22T23:55:20,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720563) roll requested 2024-11-22T23:55:20,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720571 2024-11-22T23:55:20,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720571 2024-11-22T23:55:20,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720571) roll requested 2024-11-22T23:55:20,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720579 2024-11-22T23:55:20,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720579 2024-11-22T23:55:20,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720579) roll requested 2024-11-22T23:55:20,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720585 2024-11-22T23:55:20,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720585 2024-11-22T23:55:20,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720585) roll requested 2024-11-22T23:55:20,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720592 2024-11-22T23:55:20,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720592 2024-11-22T23:55:20,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720592) roll requested 2024-11-22T23:55:20,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720599 2024-11-22T23:55:20,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720599 2024-11-22T23:55:20,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720599) roll requested 2024-11-22T23:55:20,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720607 2024-11-22T23:55:20,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720607 2024-11-22T23:55:20,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720607) roll requested 2024-11-22T23:55:20,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720614 2024-11-22T23:55:20,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720614 2024-11-22T23:55:20,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720614) roll requested 2024-11-22T23:55:20,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720621 2024-11-22T23:55:20,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720621 2024-11-22T23:55:20,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720621) roll requested 2024-11-22T23:55:20,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720629 2024-11-22T23:55:20,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720629 2024-11-22T23:55:20,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720629) roll requested 2024-11-22T23:55:20,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720636 2024-11-22T23:55:20,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720636 2024-11-22T23:55:20,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720636) roll requested 2024-11-22T23:55:20,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720645 2024-11-22T23:55:20,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720645 2024-11-22T23:55:20,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720645) roll requested 2024-11-22T23:55:20,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720652 2024-11-22T23:55:20,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720652 2024-11-22T23:55:20,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720652) roll requested 2024-11-22T23:55:20,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720660 2024-11-22T23:55:20,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720660 2024-11-22T23:55:20,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720660) roll requested 2024-11-22T23:55:20,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720667 2024-11-22T23:55:20,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720667 2024-11-22T23:55:20,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720667) roll requested 2024-11-22T23:55:20,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720674 2024-11-22T23:55:20,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720674 2024-11-22T23:55:20,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720674) roll requested 2024-11-22T23:55:20,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720680 2024-11-22T23:55:20,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720680 2024-11-22T23:55:20,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720680) roll requested 2024-11-22T23:55:20,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720687 2024-11-22T23:55:20,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720687 2024-11-22T23:55:20,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720687) roll requested 2024-11-22T23:55:20,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720694 2024-11-22T23:55:20,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720694 2024-11-22T23:55:20,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720694) roll requested 2024-11-22T23:55:20,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720702 2024-11-22T23:55:20,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720702 2024-11-22T23:55:20,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720702) roll requested 2024-11-22T23:55:20,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720709 2024-11-22T23:55:20,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720709 2024-11-22T23:55:20,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720709) roll requested 2024-11-22T23:55:20,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720716 2024-11-22T23:55:20,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720716 2024-11-22T23:55:20,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720716) roll requested 2024-11-22T23:55:20,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720723 2024-11-22T23:55:20,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720723 2024-11-22T23:55:20,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720723) roll requested 2024-11-22T23:55:20,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720730 2024-11-22T23:55:20,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720730 2024-11-22T23:55:20,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720730) roll requested 2024-11-22T23:55:20,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720737 2024-11-22T23:55:20,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720737 2024-11-22T23:55:20,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720737) roll requested 2024-11-22T23:55:20,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720745 2024-11-22T23:55:20,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720745 2024-11-22T23:55:20,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720745) roll requested 2024-11-22T23:55:20,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720752 2024-11-22T23:55:20,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720752 2024-11-22T23:55:20,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720752) roll requested 2024-11-22T23:55:20,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720760 2024-11-22T23:55:20,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720760 2024-11-22T23:55:20,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720760) roll requested 2024-11-22T23:55:20,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720767 2024-11-22T23:55:20,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720767 2024-11-22T23:55:20,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720767) roll requested 2024-11-22T23:55:20,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720774 2024-11-22T23:55:20,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720774 2024-11-22T23:55:20,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720774) roll requested 2024-11-22T23:55:20,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720782 2024-11-22T23:55:20,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720782 2024-11-22T23:55:20,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720782) roll requested 2024-11-22T23:55:20,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720789 2024-11-22T23:55:20,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720789 2024-11-22T23:55:20,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720789) roll requested 2024-11-22T23:55:20,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720799 2024-11-22T23:55:20,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720799 2024-11-22T23:55:20,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,806 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720799) roll requested 2024-11-22T23:55:20,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720806 2024-11-22T23:55:20,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720806 2024-11-22T23:55:20,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720806) roll requested 2024-11-22T23:55:20,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720813 2024-11-22T23:55:20,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720813 2024-11-22T23:55:20,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720813) roll requested 2024-11-22T23:55:20,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720821 2024-11-22T23:55:20,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720821 2024-11-22T23:55:20,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720821) roll requested 2024-11-22T23:55:20,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720828 2024-11-22T23:55:20,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720828 2024-11-22T23:55:20,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720828) roll requested 2024-11-22T23:55:20,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720835 2024-11-22T23:55:20,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720835 2024-11-22T23:55:20,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720835) roll requested 2024-11-22T23:55:20,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720842 2024-11-22T23:55:20,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720842 2024-11-22T23:55:20,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720842) roll requested 2024-11-22T23:55:20,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720850 2024-11-22T23:55:20,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720850 2024-11-22T23:55:20,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720850) roll requested 2024-11-22T23:55:20,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720858 2024-11-22T23:55:20,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720858 2024-11-22T23:55:20,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720858) roll requested 2024-11-22T23:55:20,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720865 2024-11-22T23:55:20,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720865 2024-11-22T23:55:20,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720865) roll requested 2024-11-22T23:55:20,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720872 2024-11-22T23:55:20,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720872 2024-11-22T23:55:20,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720872) roll requested 2024-11-22T23:55:20,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720879 2024-11-22T23:55:20,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720879 2024-11-22T23:55:20,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720879) roll requested 2024-11-22T23:55:20,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720890 2024-11-22T23:55:20,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720890 2024-11-22T23:55:20,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720890) roll requested 2024-11-22T23:55:20,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720897 2024-11-22T23:55:20,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720897 2024-11-22T23:55:20,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720897) roll requested 2024-11-22T23:55:20,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720905 2024-11-22T23:55:20,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720905 2024-11-22T23:55:20,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720905) roll requested 2024-11-22T23:55:20,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720912 2024-11-22T23:55:20,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720912 2024-11-22T23:55:20,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720912) roll requested 2024-11-22T23:55:20,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720925 2024-11-22T23:55:20,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720925 2024-11-22T23:55:20,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720925) roll requested 2024-11-22T23:55:20,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720933 2024-11-22T23:55:20,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720933 2024-11-22T23:55:20,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720933) roll requested 2024-11-22T23:55:20,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720944 2024-11-22T23:55:20,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720944 2024-11-22T23:55:20,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720944) roll requested 2024-11-22T23:55:20,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720952 2024-11-22T23:55:20,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720952 2024-11-22T23:55:20,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720952) roll requested 2024-11-22T23:55:20,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720971 2024-11-22T23:55:20,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720971 2024-11-22T23:55:20,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720971) roll requested 2024-11-22T23:55:20,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720979 2024-11-22T23:55:20,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720979 2024-11-22T23:55:20,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720979) roll requested 2024-11-22T23:55:20,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720986 2024-11-22T23:55:20,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720986 2024-11-22T23:55:20,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:20,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:20,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720986) roll requested 2024-11-22T23:55:20,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319720993 2024-11-22T23:55:20,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:20,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720993 2024-11-22T23:55:20,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:20,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:20,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319720993) roll requested 2024-11-22T23:55:21,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721000 2024-11-22T23:55:21,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319720993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721000 2024-11-22T23:55:21,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721000) roll requested 2024-11-22T23:55:21,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721007 2024-11-22T23:55:21,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721007 2024-11-22T23:55:21,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721007) roll requested 2024-11-22T23:55:21,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721014 2024-11-22T23:55:21,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721014 2024-11-22T23:55:21,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721014) roll requested 2024-11-22T23:55:21,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721021 2024-11-22T23:55:21,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721021 2024-11-22T23:55:21,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721021) roll requested 2024-11-22T23:55:21,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721028 2024-11-22T23:55:21,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721028 2024-11-22T23:55:21,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721028) roll requested 2024-11-22T23:55:21,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721034 2024-11-22T23:55:21,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721034 2024-11-22T23:55:21,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721034) roll requested 2024-11-22T23:55:21,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721041 2024-11-22T23:55:21,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721041 2024-11-22T23:55:21,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721041) roll requested 2024-11-22T23:55:21,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721048 2024-11-22T23:55:21,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721048 2024-11-22T23:55:21,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721048) roll requested 2024-11-22T23:55:21,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721055 2024-11-22T23:55:21,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721055 2024-11-22T23:55:21,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721055) roll requested 2024-11-22T23:55:21,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721062 2024-11-22T23:55:21,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721062 2024-11-22T23:55:21,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721062) roll requested 2024-11-22T23:55:21,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721069 2024-11-22T23:55:21,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721069 2024-11-22T23:55:21,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721069) roll requested 2024-11-22T23:55:21,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721076 2024-11-22T23:55:21,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721076 2024-11-22T23:55:21,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721076) roll requested 2024-11-22T23:55:21,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721083 2024-11-22T23:55:21,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721083 2024-11-22T23:55:21,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721083) roll requested 2024-11-22T23:55:21,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721091 2024-11-22T23:55:21,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721083 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721091 2024-11-22T23:55:21,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721091) roll requested 2024-11-22T23:55:21,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721097 2024-11-22T23:55:21,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721097 2024-11-22T23:55:21,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721097) roll requested 2024-11-22T23:55:21,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721105 2024-11-22T23:55:21,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721105 2024-11-22T23:55:21,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721105) roll requested 2024-11-22T23:55:21,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721113 2024-11-22T23:55:21,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721113 2024-11-22T23:55:21,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721113) roll requested 2024-11-22T23:55:21,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721119 2024-11-22T23:55:21,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721119 2024-11-22T23:55:21,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721119) roll requested 2024-11-22T23:55:21,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721127 2024-11-22T23:55:21,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721127 2024-11-22T23:55:21,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721127) roll requested 2024-11-22T23:55:21,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721133 2024-11-22T23:55:21,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721133 2024-11-22T23:55:21,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721133) roll requested 2024-11-22T23:55:21,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721140 2024-11-22T23:55:21,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721140 2024-11-22T23:55:21,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721140) roll requested 2024-11-22T23:55:21,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721147 2024-11-22T23:55:21,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721147 2024-11-22T23:55:21,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721147) roll requested 2024-11-22T23:55:21,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721155 2024-11-22T23:55:21,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721155 2024-11-22T23:55:21,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721155) roll requested 2024-11-22T23:55:21,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721162 2024-11-22T23:55:21,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721162 2024-11-22T23:55:21,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721162) roll requested 2024-11-22T23:55:21,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721169 2024-11-22T23:55:21,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721169 2024-11-22T23:55:21,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721169) roll requested 2024-11-22T23:55:21,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721176 2024-11-22T23:55:21,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721176 2024-11-22T23:55:21,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721176) roll requested 2024-11-22T23:55:21,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721183 2024-11-22T23:55:21,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721183 2024-11-22T23:55:21,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721183) roll requested 2024-11-22T23:55:21,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721190 2024-11-22T23:55:21,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721190 2024-11-22T23:55:21,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721190) roll requested 2024-11-22T23:55:21,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721197 2024-11-22T23:55:21,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721197 2024-11-22T23:55:21,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721197) roll requested 2024-11-22T23:55:21,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721204 2024-11-22T23:55:21,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721204 2024-11-22T23:55:21,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721204) roll requested 2024-11-22T23:55:21,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721211 2024-11-22T23:55:21,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721211 2024-11-22T23:55:21,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721211) roll requested 2024-11-22T23:55:21,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721219 2024-11-22T23:55:21,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721219 2024-11-22T23:55:21,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721219) roll requested 2024-11-22T23:55:21,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721226 2024-11-22T23:55:21,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721226 2024-11-22T23:55:21,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721226) roll requested 2024-11-22T23:55:21,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721232 2024-11-22T23:55:21,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721232 2024-11-22T23:55:21,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721232) roll requested 2024-11-22T23:55:21,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721240 2024-11-22T23:55:21,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721240 2024-11-22T23:55:21,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721240) roll requested 2024-11-22T23:55:21,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721247 2024-11-22T23:55:21,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721247 2024-11-22T23:55:21,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721247) roll requested 2024-11-22T23:55:21,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721254 2024-11-22T23:55:21,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721254 2024-11-22T23:55:21,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721254) roll requested 2024-11-22T23:55:21,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721261 2024-11-22T23:55:21,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721261 2024-11-22T23:55:21,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721261) roll requested 2024-11-22T23:55:21,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721269 2024-11-22T23:55:21,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721269 2024-11-22T23:55:21,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721269) roll requested 2024-11-22T23:55:21,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721275 2024-11-22T23:55:21,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721275 2024-11-22T23:55:21,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721275) roll requested 2024-11-22T23:55:21,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721282 2024-11-22T23:55:21,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721282 2024-11-22T23:55:21,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721282) roll requested 2024-11-22T23:55:21,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721290 2024-11-22T23:55:21,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721290 2024-11-22T23:55:21,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721290) roll requested 2024-11-22T23:55:21,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721299 2024-11-22T23:55:21,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721299 2024-11-22T23:55:21,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721299) roll requested 2024-11-22T23:55:21,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721305 2024-11-22T23:55:21,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721305 2024-11-22T23:55:21,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721305) roll requested 2024-11-22T23:55:21,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721312 2024-11-22T23:55:21,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721312 2024-11-22T23:55:21,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721312) roll requested 2024-11-22T23:55:21,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721319 2024-11-22T23:55:21,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721319 2024-11-22T23:55:21,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721319) roll requested 2024-11-22T23:55:21,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721327 2024-11-22T23:55:21,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721327 2024-11-22T23:55:21,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721327) roll requested 2024-11-22T23:55:21,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721334 2024-11-22T23:55:21,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721334 2024-11-22T23:55:21,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721334) roll requested 2024-11-22T23:55:21,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721341 2024-11-22T23:55:21,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721341 2024-11-22T23:55:21,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721341) roll requested 2024-11-22T23:55:21,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721348 2024-11-22T23:55:21,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721348 2024-11-22T23:55:21,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721348) roll requested 2024-11-22T23:55:21,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721355 2024-11-22T23:55:21,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721355 2024-11-22T23:55:21,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721355) roll requested 2024-11-22T23:55:21,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721361 2024-11-22T23:55:21,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721361 2024-11-22T23:55:21,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721361) roll requested 2024-11-22T23:55:21,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721368 2024-11-22T23:55:21,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721368 2024-11-22T23:55:21,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721368) roll requested 2024-11-22T23:55:21,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721375 2024-11-22T23:55:21,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721375 2024-11-22T23:55:21,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721375) roll requested 2024-11-22T23:55:21,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721385 2024-11-22T23:55:21,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721385 2024-11-22T23:55:21,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721385) roll requested 2024-11-22T23:55:21,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721392 2024-11-22T23:55:21,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721392 2024-11-22T23:55:21,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721392) roll requested 2024-11-22T23:55:21,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721399 2024-11-22T23:55:21,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721399 2024-11-22T23:55:21,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721399) roll requested 2024-11-22T23:55:21,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721406 2024-11-22T23:55:21,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721406 2024-11-22T23:55:21,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721406) roll requested 2024-11-22T23:55:21,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721412 2024-11-22T23:55:21,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721412 2024-11-22T23:55:21,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721412) roll requested 2024-11-22T23:55:21,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721419 2024-11-22T23:55:21,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721419 2024-11-22T23:55:21,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721419) roll requested 2024-11-22T23:55:21,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721426 2024-11-22T23:55:21,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721426 2024-11-22T23:55:21,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721426) roll requested 2024-11-22T23:55:21,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721432 2024-11-22T23:55:21,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721432 2024-11-22T23:55:21,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721432) roll requested 2024-11-22T23:55:21,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721439 2024-11-22T23:55:21,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721432 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721439 2024-11-22T23:55:21,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721439) roll requested 2024-11-22T23:55:21,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721446 2024-11-22T23:55:21,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721446 2024-11-22T23:55:21,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721446) roll requested 2024-11-22T23:55:21,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721453 2024-11-22T23:55:21,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721453 2024-11-22T23:55:21,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721453) roll requested 2024-11-22T23:55:21,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721460 2024-11-22T23:55:21,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721460 2024-11-22T23:55:21,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721460) roll requested 2024-11-22T23:55:21,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721467 2024-11-22T23:55:21,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721467 2024-11-22T23:55:21,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721467) roll requested 2024-11-22T23:55:21,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721474 2024-11-22T23:55:21,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721474 2024-11-22T23:55:21,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721474) roll requested 2024-11-22T23:55:21,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721481 2024-11-22T23:55:21,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721481 2024-11-22T23:55:21,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721481) roll requested 2024-11-22T23:55:21,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721488 2024-11-22T23:55:21,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721488 2024-11-22T23:55:21,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721488) roll requested 2024-11-22T23:55:21,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721495 2024-11-22T23:55:21,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721495 2024-11-22T23:55:21,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721495) roll requested 2024-11-22T23:55:21,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721502 2024-11-22T23:55:21,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721502 2024-11-22T23:55:21,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721502) roll requested 2024-11-22T23:55:21,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721509 2024-11-22T23:55:21,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721509 2024-11-22T23:55:21,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721509) roll requested 2024-11-22T23:55:21,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721516 2024-11-22T23:55:21,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721516 2024-11-22T23:55:21,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721516) roll requested 2024-11-22T23:55:21,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721523 2024-11-22T23:55:21,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721523 2024-11-22T23:55:21,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721523) roll requested 2024-11-22T23:55:21,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721530 2024-11-22T23:55:21,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721530 2024-11-22T23:55:21,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721530) roll requested 2024-11-22T23:55:21,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721537 2024-11-22T23:55:21,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721537 2024-11-22T23:55:21,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721537) roll requested 2024-11-22T23:55:21,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721544 2024-11-22T23:55:21,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721544 2024-11-22T23:55:21,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721544) roll requested 2024-11-22T23:55:21,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721551 2024-11-22T23:55:21,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721551 2024-11-22T23:55:21,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721551) roll requested 2024-11-22T23:55:21,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721558 2024-11-22T23:55:21,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721558 2024-11-22T23:55:21,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,565 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721558) roll requested 2024-11-22T23:55:21,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721565 2024-11-22T23:55:21,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721565 2024-11-22T23:55:21,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721565) roll requested 2024-11-22T23:55:21,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721573 2024-11-22T23:55:21,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721573 2024-11-22T23:55:21,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721573) roll requested 2024-11-22T23:55:21,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721580 2024-11-22T23:55:21,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721580 2024-11-22T23:55:21,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721580) roll requested 2024-11-22T23:55:21,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721587 2024-11-22T23:55:21,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721587 2024-11-22T23:55:21,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721587) roll requested 2024-11-22T23:55:21,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721594 2024-11-22T23:55:21,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721587 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721594 2024-11-22T23:55:21,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721594) roll requested 2024-11-22T23:55:21,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721600 2024-11-22T23:55:21,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721600 2024-11-22T23:55:21,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721600) roll requested 2024-11-22T23:55:21,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721608 2024-11-22T23:55:21,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721608 2024-11-22T23:55:21,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721608) roll requested 2024-11-22T23:55:21,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721615 2024-11-22T23:55:21,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721615 2024-11-22T23:55:21,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721615) roll requested 2024-11-22T23:55:21,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721624 2024-11-22T23:55:21,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721624 2024-11-22T23:55:21,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721624) roll requested 2024-11-22T23:55:21,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721631 2024-11-22T23:55:21,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721631 2024-11-22T23:55:21,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721631) roll requested 2024-11-22T23:55:21,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721638 2024-11-22T23:55:21,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721638 2024-11-22T23:55:21,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721638) roll requested 2024-11-22T23:55:21,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721645 2024-11-22T23:55:21,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721645 2024-11-22T23:55:21,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721645) roll requested 2024-11-22T23:55:21,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721653 2024-11-22T23:55:21,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721653 2024-11-22T23:55:21,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721653) roll requested 2024-11-22T23:55:21,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721660 2024-11-22T23:55:21,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721660 2024-11-22T23:55:21,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721660) roll requested 2024-11-22T23:55:21,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721667 2024-11-22T23:55:21,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721667 2024-11-22T23:55:21,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721667) roll requested 2024-11-22T23:55:21,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721674 2024-11-22T23:55:21,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721674 2024-11-22T23:55:21,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721674) roll requested 2024-11-22T23:55:21,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721681 2024-11-22T23:55:21,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721681 2024-11-22T23:55:21,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721681) roll requested 2024-11-22T23:55:21,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721687 2024-11-22T23:55:21,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721687 2024-11-22T23:55:21,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721687) roll requested 2024-11-22T23:55:21,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721695 2024-11-22T23:55:21,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721695 2024-11-22T23:55:21,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721695) roll requested 2024-11-22T23:55:21,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721701 2024-11-22T23:55:21,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721701 2024-11-22T23:55:21,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721701) roll requested 2024-11-22T23:55:21,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721709 2024-11-22T23:55:21,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721709 2024-11-22T23:55:21,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721709) roll requested 2024-11-22T23:55:21,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721715 2024-11-22T23:55:21,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721715 2024-11-22T23:55:21,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721715) roll requested 2024-11-22T23:55:21,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721722 2024-11-22T23:55:21,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721722 2024-11-22T23:55:21,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721722) roll requested 2024-11-22T23:55:21,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721730 2024-11-22T23:55:21,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721730 2024-11-22T23:55:21,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721730) roll requested 2024-11-22T23:55:21,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721737 2024-11-22T23:55:21,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721737 2024-11-22T23:55:21,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721737) roll requested 2024-11-22T23:55:21,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721744 2024-11-22T23:55:21,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721744 2024-11-22T23:55:21,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721744) roll requested 2024-11-22T23:55:21,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721751 2024-11-22T23:55:21,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721751 2024-11-22T23:55:21,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721751) roll requested 2024-11-22T23:55:21,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721758 2024-11-22T23:55:21,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721758 2024-11-22T23:55:21,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721758) roll requested 2024-11-22T23:55:21,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721766 2024-11-22T23:55:21,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721766 2024-11-22T23:55:21,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721766) roll requested 2024-11-22T23:55:21,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721776 2024-11-22T23:55:21,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721776 2024-11-22T23:55:21,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721776) roll requested 2024-11-22T23:55:21,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721785 2024-11-22T23:55:21,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721785 2024-11-22T23:55:21,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721785) roll requested 2024-11-22T23:55:21,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721792 2024-11-22T23:55:21,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721792 2024-11-22T23:55:21,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721792) roll requested 2024-11-22T23:55:21,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721801 2024-11-22T23:55:21,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721801 2024-11-22T23:55:21,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721801) roll requested 2024-11-22T23:55:21,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721808 2024-11-22T23:55:21,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721808 2024-11-22T23:55:21,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721808) roll requested 2024-11-22T23:55:21,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721816 2024-11-22T23:55:21,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721816 2024-11-22T23:55:21,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721816) roll requested 2024-11-22T23:55:21,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721823 2024-11-22T23:55:21,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721823 2024-11-22T23:55:21,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721823) roll requested 2024-11-22T23:55:21,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721832 2024-11-22T23:55:21,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721832 2024-11-22T23:55:21,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721832) roll requested 2024-11-22T23:55:21,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721840 2024-11-22T23:55:21,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721840 2024-11-22T23:55:21,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721840) roll requested 2024-11-22T23:55:21,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721853 2024-11-22T23:55:21,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721853 2024-11-22T23:55:21,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721853) roll requested 2024-11-22T23:55:21,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721861 2024-11-22T23:55:21,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721861 2024-11-22T23:55:21,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,868 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721861) roll requested 2024-11-22T23:55:21,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721868 2024-11-22T23:55:21,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721868 2024-11-22T23:55:21,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721868) roll requested 2024-11-22T23:55:21,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721876 2024-11-22T23:55:21,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721868 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721876 2024-11-22T23:55:21,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721876) roll requested 2024-11-22T23:55:21,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721883 2024-11-22T23:55:21,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721883 2024-11-22T23:55:21,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721883) roll requested 2024-11-22T23:55:21,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721891 2024-11-22T23:55:21,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721891 2024-11-22T23:55:21,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721891) roll requested 2024-11-22T23:55:21,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721899 2024-11-22T23:55:21,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721899 2024-11-22T23:55:21,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721899) roll requested 2024-11-22T23:55:21,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721906 2024-11-22T23:55:21,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721906 2024-11-22T23:55:21,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721906) roll requested 2024-11-22T23:55:21,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721914 2024-11-22T23:55:21,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721914 2024-11-22T23:55:21,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721914) roll requested 2024-11-22T23:55:21,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721922 2024-11-22T23:55:21,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721922 2024-11-22T23:55:21,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721922) roll requested 2024-11-22T23:55:21,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721931 2024-11-22T23:55:21,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721931 2024-11-22T23:55:21,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721931) roll requested 2024-11-22T23:55:21,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721939 2024-11-22T23:55:21,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721939 2024-11-22T23:55:21,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721939) roll requested 2024-11-22T23:55:21,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721946 2024-11-22T23:55:21,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721946 2024-11-22T23:55:21,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721946) roll requested 2024-11-22T23:55:21,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721953 2024-11-22T23:55:21,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721953 2024-11-22T23:55:21,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721953) roll requested 2024-11-22T23:55:21,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721960 2024-11-22T23:55:21,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721960 2024-11-22T23:55:21,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721960) roll requested 2024-11-22T23:55:21,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721967 2024-11-22T23:55:21,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721967 2024-11-22T23:55:21,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721967) roll requested 2024-11-22T23:55:21,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721974 2024-11-22T23:55:21,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721974 2024-11-22T23:55:21,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721974) roll requested 2024-11-22T23:55:21,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721981 2024-11-22T23:55:21,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721981 2024-11-22T23:55:21,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721981) roll requested 2024-11-22T23:55:21,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721988 2024-11-22T23:55:21,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:21,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721988 2024-11-22T23:55:21,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:21,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:21,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:21,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:21,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721988) roll requested 2024-11-22T23:55:21,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319721994 2024-11-22T23:55:22,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721994 2024-11-22T23:55:22,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319721994) roll requested 2024-11-22T23:55:22,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722001 2024-11-22T23:55:22,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319721994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722001 2024-11-22T23:55:22,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722001) roll requested 2024-11-22T23:55:22,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722008 2024-11-22T23:55:22,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722008 2024-11-22T23:55:22,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722008) roll requested 2024-11-22T23:55:22,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722015 2024-11-22T23:55:22,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722015 2024-11-22T23:55:22,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722015) roll requested 2024-11-22T23:55:22,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722023 2024-11-22T23:55:22,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722023 2024-11-22T23:55:22,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722023) roll requested 2024-11-22T23:55:22,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722030 2024-11-22T23:55:22,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722030 2024-11-22T23:55:22,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722030) roll requested 2024-11-22T23:55:22,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722037 2024-11-22T23:55:22,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722037 2024-11-22T23:55:22,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722037) roll requested 2024-11-22T23:55:22,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722044 2024-11-22T23:55:22,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722044 2024-11-22T23:55:22,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722044) roll requested 2024-11-22T23:55:22,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722051 2024-11-22T23:55:22,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722051 2024-11-22T23:55:22,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722051) roll requested 2024-11-22T23:55:22,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722058 2024-11-22T23:55:22,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722058 2024-11-22T23:55:22,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722058) roll requested 2024-11-22T23:55:22,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722065 2024-11-22T23:55:22,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722065 2024-11-22T23:55:22,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722065) roll requested 2024-11-22T23:55:22,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722073 2024-11-22T23:55:22,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722073 2024-11-22T23:55:22,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722073) roll requested 2024-11-22T23:55:22,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722080 2024-11-22T23:55:22,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722080 2024-11-22T23:55:22,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722080) roll requested 2024-11-22T23:55:22,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722087 2024-11-22T23:55:22,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722087 2024-11-22T23:55:22,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722087) roll requested 2024-11-22T23:55:22,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722094 2024-11-22T23:55:22,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722094 2024-11-22T23:55:22,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722094) roll requested 2024-11-22T23:55:22,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722101 2024-11-22T23:55:22,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722101 2024-11-22T23:55:22,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722101) roll requested 2024-11-22T23:55:22,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722108 2024-11-22T23:55:22,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722108 2024-11-22T23:55:22,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722108) roll requested 2024-11-22T23:55:22,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722115 2024-11-22T23:55:22,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722115 2024-11-22T23:55:22,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722115) roll requested 2024-11-22T23:55:22,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722123 2024-11-22T23:55:22,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722123 2024-11-22T23:55:22,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722123) roll requested 2024-11-22T23:55:22,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722130 2024-11-22T23:55:22,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722130 2024-11-22T23:55:22,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722130) roll requested 2024-11-22T23:55:22,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722137 2024-11-22T23:55:22,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722137 2024-11-22T23:55:22,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722137) roll requested 2024-11-22T23:55:22,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722145 2024-11-22T23:55:22,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722145 2024-11-22T23:55:22,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722145) roll requested 2024-11-22T23:55:22,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722152 2024-11-22T23:55:22,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722152 2024-11-22T23:55:22,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722152) roll requested 2024-11-22T23:55:22,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722159 2024-11-22T23:55:22,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722159 2024-11-22T23:55:22,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722159) roll requested 2024-11-22T23:55:22,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722166 2024-11-22T23:55:22,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722166 2024-11-22T23:55:22,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722166) roll requested 2024-11-22T23:55:22,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722182 2024-11-22T23:55:22,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722182 2024-11-22T23:55:22,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722182) roll requested 2024-11-22T23:55:22,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722195 2024-11-22T23:55:22,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722195 2024-11-22T23:55:22,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722195) roll requested 2024-11-22T23:55:22,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722201 2024-11-22T23:55:22,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722201 2024-11-22T23:55:22,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722201) roll requested 2024-11-22T23:55:22,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722208 2024-11-22T23:55:22,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722208 2024-11-22T23:55:22,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722208) roll requested 2024-11-22T23:55:22,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722215 2024-11-22T23:55:22,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722215 2024-11-22T23:55:22,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722215) roll requested 2024-11-22T23:55:22,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722222 2024-11-22T23:55:22,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722222 2024-11-22T23:55:22,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722222) roll requested 2024-11-22T23:55:22,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722229 2024-11-22T23:55:22,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722229 2024-11-22T23:55:22,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722229) roll requested 2024-11-22T23:55:22,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722236 2024-11-22T23:55:22,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722236 2024-11-22T23:55:22,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722236) roll requested 2024-11-22T23:55:22,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722243 2024-11-22T23:55:22,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722243 2024-11-22T23:55:22,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722243) roll requested 2024-11-22T23:55:22,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722249 2024-11-22T23:55:22,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722249 2024-11-22T23:55:22,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722249) roll requested 2024-11-22T23:55:22,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722256 2024-11-22T23:55:22,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722256 2024-11-22T23:55:22,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722256) roll requested 2024-11-22T23:55:22,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722263 2024-11-22T23:55:22,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722263 2024-11-22T23:55:22,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722263) roll requested 2024-11-22T23:55:22,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722270 2024-11-22T23:55:22,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722270 2024-11-22T23:55:22,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722270) roll requested 2024-11-22T23:55:22,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722277 2024-11-22T23:55:22,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722277 2024-11-22T23:55:22,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722277) roll requested 2024-11-22T23:55:22,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722284 2024-11-22T23:55:22,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722284 2024-11-22T23:55:22,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722284) roll requested 2024-11-22T23:55:22,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722291 2024-11-22T23:55:22,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722291 2024-11-22T23:55:22,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722291) roll requested 2024-11-22T23:55:22,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722299 2024-11-22T23:55:22,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722299 2024-11-22T23:55:22,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722299) roll requested 2024-11-22T23:55:22,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722305 2024-11-22T23:55:22,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722305 2024-11-22T23:55:22,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722305) roll requested 2024-11-22T23:55:22,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722313 2024-11-22T23:55:22,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722313 2024-11-22T23:55:22,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722313) roll requested 2024-11-22T23:55:22,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722320 2024-11-22T23:55:22,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722320 2024-11-22T23:55:22,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722320) roll requested 2024-11-22T23:55:22,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722328 2024-11-22T23:55:22,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722328 2024-11-22T23:55:22,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722328) roll requested 2024-11-22T23:55:22,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722335 2024-11-22T23:55:22,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722335 2024-11-22T23:55:22,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722335) roll requested 2024-11-22T23:55:22,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722342 2024-11-22T23:55:22,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722342 2024-11-22T23:55:22,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722342) roll requested 2024-11-22T23:55:22,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722349 2024-11-22T23:55:22,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722349 2024-11-22T23:55:22,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722349) roll requested 2024-11-22T23:55:22,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722356 2024-11-22T23:55:22,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722356 2024-11-22T23:55:22,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722356) roll requested 2024-11-22T23:55:22,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722363 2024-11-22T23:55:22,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722363 2024-11-22T23:55:22,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722363) roll requested 2024-11-22T23:55:22,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722370 2024-11-22T23:55:22,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722370 2024-11-22T23:55:22,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722370) roll requested 2024-11-22T23:55:22,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722377 2024-11-22T23:55:22,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722377 2024-11-22T23:55:22,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722377) roll requested 2024-11-22T23:55:22,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722384 2024-11-22T23:55:22,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722384 2024-11-22T23:55:22,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722384) roll requested 2024-11-22T23:55:22,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722392 2024-11-22T23:55:22,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722392 2024-11-22T23:55:22,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722392) roll requested 2024-11-22T23:55:22,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722399 2024-11-22T23:55:22,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722399 2024-11-22T23:55:22,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722399) roll requested 2024-11-22T23:55:22,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722406 2024-11-22T23:55:22,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722406 2024-11-22T23:55:22,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722406) roll requested 2024-11-22T23:55:22,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722413 2024-11-22T23:55:22,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722413 2024-11-22T23:55:22,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722413) roll requested 2024-11-22T23:55:22,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722420 2024-11-22T23:55:22,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722420 2024-11-22T23:55:22,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722420) roll requested 2024-11-22T23:55:22,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722427 2024-11-22T23:55:22,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722427 2024-11-22T23:55:22,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722427) roll requested 2024-11-22T23:55:22,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722434 2024-11-22T23:55:22,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722434 2024-11-22T23:55:22,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722434) roll requested 2024-11-22T23:55:22,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722441 2024-11-22T23:55:22,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722441 2024-11-22T23:55:22,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722441) roll requested 2024-11-22T23:55:22,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722448 2024-11-22T23:55:22,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722448 2024-11-22T23:55:22,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722448) roll requested 2024-11-22T23:55:22,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722455 2024-11-22T23:55:22,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722455 2024-11-22T23:55:22,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722455) roll requested 2024-11-22T23:55:22,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722462 2024-11-22T23:55:22,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722462 2024-11-22T23:55:22,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722462) roll requested 2024-11-22T23:55:22,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722469 2024-11-22T23:55:22,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722469 2024-11-22T23:55:22,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722469) roll requested 2024-11-22T23:55:22,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722476 2024-11-22T23:55:22,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722476 2024-11-22T23:55:22,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722476) roll requested 2024-11-22T23:55:22,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722483 2024-11-22T23:55:22,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722483 2024-11-22T23:55:22,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722483) roll requested 2024-11-22T23:55:22,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722491 2024-11-22T23:55:22,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722491 2024-11-22T23:55:22,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722491) roll requested 2024-11-22T23:55:22,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722498 2024-11-22T23:55:22,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722498 2024-11-22T23:55:22,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722498) roll requested 2024-11-22T23:55:22,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722505 2024-11-22T23:55:22,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722505 2024-11-22T23:55:22,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722505) roll requested 2024-11-22T23:55:22,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722513 2024-11-22T23:55:22,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722513 2024-11-22T23:55:22,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722513) roll requested 2024-11-22T23:55:22,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722519 2024-11-22T23:55:22,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722519 2024-11-22T23:55:22,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722519) roll requested 2024-11-22T23:55:22,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722526 2024-11-22T23:55:22,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722526 2024-11-22T23:55:22,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722526) roll requested 2024-11-22T23:55:22,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722534 2024-11-22T23:55:22,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722534 2024-11-22T23:55:22,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722534) roll requested 2024-11-22T23:55:22,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722541 2024-11-22T23:55:22,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722541 2024-11-22T23:55:22,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722541) roll requested 2024-11-22T23:55:22,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722549 2024-11-22T23:55:22,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722549 2024-11-22T23:55:22,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722549) roll requested 2024-11-22T23:55:22,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722557 2024-11-22T23:55:22,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722557 2024-11-22T23:55:22,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722557) roll requested 2024-11-22T23:55:22,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722564 2024-11-22T23:55:22,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722564 2024-11-22T23:55:22,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722564) roll requested 2024-11-22T23:55:22,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722571 2024-11-22T23:55:22,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722571 2024-11-22T23:55:22,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722571) roll requested 2024-11-22T23:55:22,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722579 2024-11-22T23:55:22,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722579 2024-11-22T23:55:22,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722579) roll requested 2024-11-22T23:55:22,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722586 2024-11-22T23:55:22,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722586 2024-11-22T23:55:22,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722586) roll requested 2024-11-22T23:55:22,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722594 2024-11-22T23:55:22,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722594 2024-11-22T23:55:22,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722594) roll requested 2024-11-22T23:55:22,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722601 2024-11-22T23:55:22,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722601 2024-11-22T23:55:22,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722601) roll requested 2024-11-22T23:55:22,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722608 2024-11-22T23:55:22,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722608 2024-11-22T23:55:22,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722608) roll requested 2024-11-22T23:55:22,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722615 2024-11-22T23:55:22,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722615 2024-11-22T23:55:22,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722615) roll requested 2024-11-22T23:55:22,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722623 2024-11-22T23:55:22,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722623 2024-11-22T23:55:22,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722623) roll requested 2024-11-22T23:55:22,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722631 2024-11-22T23:55:22,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722631 2024-11-22T23:55:22,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722631) roll requested 2024-11-22T23:55:22,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722638 2024-11-22T23:55:22,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722638 2024-11-22T23:55:22,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722638) roll requested 2024-11-22T23:55:22,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722651 2024-11-22T23:55:22,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722651 2024-11-22T23:55:22,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722651) roll requested 2024-11-22T23:55:22,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722658 2024-11-22T23:55:22,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722658 2024-11-22T23:55:22,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722658) roll requested 2024-11-22T23:55:22,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722666 2024-11-22T23:55:22,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722666 2024-11-22T23:55:22,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722666) roll requested 2024-11-22T23:55:22,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722673 2024-11-22T23:55:22,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722673 2024-11-22T23:55:22,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722673) roll requested 2024-11-22T23:55:22,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722680 2024-11-22T23:55:22,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722680 2024-11-22T23:55:22,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722680) roll requested 2024-11-22T23:55:22,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722687 2024-11-22T23:55:22,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722687 2024-11-22T23:55:22,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722687) roll requested 2024-11-22T23:55:22,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722694 2024-11-22T23:55:22,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722694 2024-11-22T23:55:22,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722694) roll requested 2024-11-22T23:55:22,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722701 2024-11-22T23:55:22,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722701 2024-11-22T23:55:22,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722701) roll requested 2024-11-22T23:55:22,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722709 2024-11-22T23:55:22,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722709 2024-11-22T23:55:22,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722709) roll requested 2024-11-22T23:55:22,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722716 2024-11-22T23:55:22,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722716 2024-11-22T23:55:22,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722716) roll requested 2024-11-22T23:55:22,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722723 2024-11-22T23:55:22,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722723 2024-11-22T23:55:22,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722723) roll requested 2024-11-22T23:55:22,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722730 2024-11-22T23:55:22,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722730 2024-11-22T23:55:22,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722730) roll requested 2024-11-22T23:55:22,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722738 2024-11-22T23:55:22,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722738 2024-11-22T23:55:22,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722738) roll requested 2024-11-22T23:55:22,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722745 2024-11-22T23:55:22,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722745 2024-11-22T23:55:22,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722745) roll requested 2024-11-22T23:55:22,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722753 2024-11-22T23:55:22,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722753 2024-11-22T23:55:22,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722753) roll requested 2024-11-22T23:55:22,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722761 2024-11-22T23:55:22,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722761 2024-11-22T23:55:22,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722761) roll requested 2024-11-22T23:55:22,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722768 2024-11-22T23:55:22,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722768 2024-11-22T23:55:22,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722768) roll requested 2024-11-22T23:55:22,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722775 2024-11-22T23:55:22,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722775 2024-11-22T23:55:22,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722775) roll requested 2024-11-22T23:55:22,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722782 2024-11-22T23:55:22,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722782 2024-11-22T23:55:22,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722782) roll requested 2024-11-22T23:55:22,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722790 2024-11-22T23:55:22,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722790 2024-11-22T23:55:22,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722790) roll requested 2024-11-22T23:55:22,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722796 2024-11-22T23:55:22,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722796 2024-11-22T23:55:22,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722796) roll requested 2024-11-22T23:55:22,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722803 2024-11-22T23:55:22,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722803 2024-11-22T23:55:22,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722803) roll requested 2024-11-22T23:55:22,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722810 2024-11-22T23:55:22,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722810 2024-11-22T23:55:22,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722810) roll requested 2024-11-22T23:55:22,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722817 2024-11-22T23:55:22,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722817 2024-11-22T23:55:22,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722817) roll requested 2024-11-22T23:55:22,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722824 2024-11-22T23:55:22,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722824 2024-11-22T23:55:22,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722824) roll requested 2024-11-22T23:55:22,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722831 2024-11-22T23:55:22,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722831 2024-11-22T23:55:22,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722831) roll requested 2024-11-22T23:55:22,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722838 2024-11-22T23:55:22,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722838 2024-11-22T23:55:22,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722838) roll requested 2024-11-22T23:55:22,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722845 2024-11-22T23:55:22,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722845 2024-11-22T23:55:22,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722845) roll requested 2024-11-22T23:55:22,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722852 2024-11-22T23:55:22,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722852 2024-11-22T23:55:22,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722852) roll requested 2024-11-22T23:55:22,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722860 2024-11-22T23:55:22,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722860 2024-11-22T23:55:22,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722860) roll requested 2024-11-22T23:55:22,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722867 2024-11-22T23:55:22,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722867 2024-11-22T23:55:22,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722867) roll requested 2024-11-22T23:55:22,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722874 2024-11-22T23:55:22,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722874 2024-11-22T23:55:22,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722874) roll requested 2024-11-22T23:55:22,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722881 2024-11-22T23:55:22,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722881 2024-11-22T23:55:22,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722881) roll requested 2024-11-22T23:55:22,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722888 2024-11-22T23:55:22,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722888 2024-11-22T23:55:22,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722888) roll requested 2024-11-22T23:55:22,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722895 2024-11-22T23:55:22,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722895 2024-11-22T23:55:22,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722895) roll requested 2024-11-22T23:55:22,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722901 2024-11-22T23:55:22,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722901 2024-11-22T23:55:22,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722901) roll requested 2024-11-22T23:55:22,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722909 2024-11-22T23:55:22,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722909 2024-11-22T23:55:22,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722909) roll requested 2024-11-22T23:55:22,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722916 2024-11-22T23:55:22,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722916 2024-11-22T23:55:22,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722916) roll requested 2024-11-22T23:55:22,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722923 2024-11-22T23:55:22,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722923 2024-11-22T23:55:22,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722923) roll requested 2024-11-22T23:55:22,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722930 2024-11-22T23:55:22,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722930 2024-11-22T23:55:22,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722930) roll requested 2024-11-22T23:55:22,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722937 2024-11-22T23:55:22,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722937 2024-11-22T23:55:22,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,944 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,944 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722937) roll requested 2024-11-22T23:55:22,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722944 2024-11-22T23:55:22,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722944 2024-11-22T23:55:22,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722944) roll requested 2024-11-22T23:55:22,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722952 2024-11-22T23:55:22,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722952 2024-11-22T23:55:22,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722952) roll requested 2024-11-22T23:55:22,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722959 2024-11-22T23:55:22,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722959 2024-11-22T23:55:22,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722959) roll requested 2024-11-22T23:55:22,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722966 2024-11-22T23:55:22,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722966 2024-11-22T23:55:22,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722966) roll requested 2024-11-22T23:55:22,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722973 2024-11-22T23:55:22,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722973 2024-11-22T23:55:22,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722973) roll requested 2024-11-22T23:55:22,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722980 2024-11-22T23:55:22,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722980 2024-11-22T23:55:22,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1283, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722980) roll requested 2024-11-22T23:55:22,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722987 2024-11-22T23:55:22,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722987 2024-11-22T23:55:22,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:22,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:22,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1284, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:22,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:22,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722987) roll requested 2024-11-22T23:55:22,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319722994 2024-11-22T23:55:22,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:22,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722994 2024-11-22T23:55:23,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1285, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319722994) roll requested 2024-11-22T23:55:23,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723000 2024-11-22T23:55:23,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319722994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723000 2024-11-22T23:55:23,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1286, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723000) roll requested 2024-11-22T23:55:23,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723007 2024-11-22T23:55:23,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723007 2024-11-22T23:55:23,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1287, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723007) roll requested 2024-11-22T23:55:23,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723014 2024-11-22T23:55:23,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723014 2024-11-22T23:55:23,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1288, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723014) roll requested 2024-11-22T23:55:23,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723022 2024-11-22T23:55:23,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723022 2024-11-22T23:55:23,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1289, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723022) roll requested 2024-11-22T23:55:23,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723029 2024-11-22T23:55:23,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723029 2024-11-22T23:55:23,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1290, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723029) roll requested 2024-11-22T23:55:23,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723036 2024-11-22T23:55:23,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723036 2024-11-22T23:55:23,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1291, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723036) roll requested 2024-11-22T23:55:23,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723043 2024-11-22T23:55:23,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723043 2024-11-22T23:55:23,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1292, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723043) roll requested 2024-11-22T23:55:23,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723050 2024-11-22T23:55:23,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723050 2024-11-22T23:55:23,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1293, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723050) roll requested 2024-11-22T23:55:23,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723057 2024-11-22T23:55:23,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723057 2024-11-22T23:55:23,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1294, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723057) roll requested 2024-11-22T23:55:23,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723064 2024-11-22T23:55:23,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723064 2024-11-22T23:55:23,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,071 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1295, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723064) roll requested 2024-11-22T23:55:23,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723071 2024-11-22T23:55:23,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723071 2024-11-22T23:55:23,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1296, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723071) roll requested 2024-11-22T23:55:23,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723079 2024-11-22T23:55:23,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723079 2024-11-22T23:55:23,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1297, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723079) roll requested 2024-11-22T23:55:23,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723086 2024-11-22T23:55:23,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723086 2024-11-22T23:55:23,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1298, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723086) roll requested 2024-11-22T23:55:23,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723093 2024-11-22T23:55:23,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723093 2024-11-22T23:55:23,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1299, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723093) roll requested 2024-11-22T23:55:23,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723100 2024-11-22T23:55:23,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723100 2024-11-22T23:55:23,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1300, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723100) roll requested 2024-11-22T23:55:23,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723107 2024-11-22T23:55:23,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723107 2024-11-22T23:55:23,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1301, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723107) roll requested 2024-11-22T23:55:23,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723115 2024-11-22T23:55:23,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723115 2024-11-22T23:55:23,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1302, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723115) roll requested 2024-11-22T23:55:23,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723122 2024-11-22T23:55:23,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723122 2024-11-22T23:55:23,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1303, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723122) roll requested 2024-11-22T23:55:23,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723129 2024-11-22T23:55:23,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723129 2024-11-22T23:55:23,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1304, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723129) roll requested 2024-11-22T23:55:23,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723136 2024-11-22T23:55:23,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723136 2024-11-22T23:55:23,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1305, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723136) roll requested 2024-11-22T23:55:23,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723144 2024-11-22T23:55:23,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723144 2024-11-22T23:55:23,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1306, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723144) roll requested 2024-11-22T23:55:23,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723151 2024-11-22T23:55:23,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723151 2024-11-22T23:55:23,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1307, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723151) roll requested 2024-11-22T23:55:23,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723160 2024-11-22T23:55:23,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723160 2024-11-22T23:55:23,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1308, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723160) roll requested 2024-11-22T23:55:23,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723167 2024-11-22T23:55:23,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723167 2024-11-22T23:55:23,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1309, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723167) roll requested 2024-11-22T23:55:23,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723174 2024-11-22T23:55:23,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723174 2024-11-22T23:55:23,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1310, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723174) roll requested 2024-11-22T23:55:23,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723181 2024-11-22T23:55:23,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723181 2024-11-22T23:55:23,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1311, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723181) roll requested 2024-11-22T23:55:23,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723188 2024-11-22T23:55:23,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723188 2024-11-22T23:55:23,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1312, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723188) roll requested 2024-11-22T23:55:23,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723195 2024-11-22T23:55:23,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723195 2024-11-22T23:55:23,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1313, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723195) roll requested 2024-11-22T23:55:23,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723202 2024-11-22T23:55:23,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723202 2024-11-22T23:55:23,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1314, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723202) roll requested 2024-11-22T23:55:23,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723209 2024-11-22T23:55:23,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723209 2024-11-22T23:55:23,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1315, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723209) roll requested 2024-11-22T23:55:23,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723216 2024-11-22T23:55:23,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723216 2024-11-22T23:55:23,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1316, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723216) roll requested 2024-11-22T23:55:23,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723224 2024-11-22T23:55:23,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723224 2024-11-22T23:55:23,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1317, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723224) roll requested 2024-11-22T23:55:23,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723231 2024-11-22T23:55:23,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723231 2024-11-22T23:55:23,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1318, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723231) roll requested 2024-11-22T23:55:23,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723239 2024-11-22T23:55:23,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723239 2024-11-22T23:55:23,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1319, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723239) roll requested 2024-11-22T23:55:23,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723246 2024-11-22T23:55:23,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723246 2024-11-22T23:55:23,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1320, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723246) roll requested 2024-11-22T23:55:23,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723253 2024-11-22T23:55:23,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723253 2024-11-22T23:55:23,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1321, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723253) roll requested 2024-11-22T23:55:23,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723261 2024-11-22T23:55:23,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723261 2024-11-22T23:55:23,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1322, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723261) roll requested 2024-11-22T23:55:23,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723268 2024-11-22T23:55:23,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723268 2024-11-22T23:55:23,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1323, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723268) roll requested 2024-11-22T23:55:23,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723275 2024-11-22T23:55:23,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723275 2024-11-22T23:55:23,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1324, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723275) roll requested 2024-11-22T23:55:23,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723282 2024-11-22T23:55:23,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723282 2024-11-22T23:55:23,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1325, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723282) roll requested 2024-11-22T23:55:23,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723290 2024-11-22T23:55:23,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723290 2024-11-22T23:55:23,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1326, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723290) roll requested 2024-11-22T23:55:23,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723297 2024-11-22T23:55:23,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723297 2024-11-22T23:55:23,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1327, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723297) roll requested 2024-11-22T23:55:23,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723304 2024-11-22T23:55:23,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723304 2024-11-22T23:55:23,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1328, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723304) roll requested 2024-11-22T23:55:23,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723311 2024-11-22T23:55:23,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723311 2024-11-22T23:55:23,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1329, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723311) roll requested 2024-11-22T23:55:23,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723318 2024-11-22T23:55:23,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723318 2024-11-22T23:55:23,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1330, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723318) roll requested 2024-11-22T23:55:23,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723326 2024-11-22T23:55:23,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723326 2024-11-22T23:55:23,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1331, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723326) roll requested 2024-11-22T23:55:23,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723333 2024-11-22T23:55:23,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723333 2024-11-22T23:55:23,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1332, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723333) roll requested 2024-11-22T23:55:23,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723341 2024-11-22T23:55:23,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723341 2024-11-22T23:55:23,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1333, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723341) roll requested 2024-11-22T23:55:23,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723348 2024-11-22T23:55:23,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723348 2024-11-22T23:55:23,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1334, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723348) roll requested 2024-11-22T23:55:23,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723361 2024-11-22T23:55:23,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723361 2024-11-22T23:55:23,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,370 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1335, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,370 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723361) roll requested 2024-11-22T23:55:23,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723370 2024-11-22T23:55:23,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723370 2024-11-22T23:55:23,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1336, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723370) roll requested 2024-11-22T23:55:23,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723382 2024-11-22T23:55:23,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723382 2024-11-22T23:55:23,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1337, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723382) roll requested 2024-11-22T23:55:23,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723389 2024-11-22T23:55:23,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723389 2024-11-22T23:55:23,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1338, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723389) roll requested 2024-11-22T23:55:23,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723396 2024-11-22T23:55:23,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723396 2024-11-22T23:55:23,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1339, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723396) roll requested 2024-11-22T23:55:23,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723404 2024-11-22T23:55:23,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723404 2024-11-22T23:55:23,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1340, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723404) roll requested 2024-11-22T23:55:23,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723412 2024-11-22T23:55:23,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723412 2024-11-22T23:55:23,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1341, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723412) roll requested 2024-11-22T23:55:23,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723420 2024-11-22T23:55:23,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723420 2024-11-22T23:55:23,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1342, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723420) roll requested 2024-11-22T23:55:23,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723428 2024-11-22T23:55:23,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723428 2024-11-22T23:55:23,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1343, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723428) roll requested 2024-11-22T23:55:23,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723437 2024-11-22T23:55:23,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723437 2024-11-22T23:55:23,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1344, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723437) roll requested 2024-11-22T23:55:23,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723445 2024-11-22T23:55:23,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723445 2024-11-22T23:55:23,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1345, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723445) roll requested 2024-11-22T23:55:23,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723459 2024-11-22T23:55:23,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723459 2024-11-22T23:55:23,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,467 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1346, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723459) roll requested 2024-11-22T23:55:23,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723467 2024-11-22T23:55:23,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723467 2024-11-22T23:55:23,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1347, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723467) roll requested 2024-11-22T23:55:23,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723475 2024-11-22T23:55:23,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723475 2024-11-22T23:55:23,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1348, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723475) roll requested 2024-11-22T23:55:23,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723482 2024-11-22T23:55:23,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723482 2024-11-22T23:55:23,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1349, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723482) roll requested 2024-11-22T23:55:23,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723489 2024-11-22T23:55:23,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723489 2024-11-22T23:55:23,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1350, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723489) roll requested 2024-11-22T23:55:23,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723496 2024-11-22T23:55:23,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723496 2024-11-22T23:55:23,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1351, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723496) roll requested 2024-11-22T23:55:23,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723504 2024-11-22T23:55:23,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723504 2024-11-22T23:55:23,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1352, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723504) roll requested 2024-11-22T23:55:23,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723511 2024-11-22T23:55:23,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723511 2024-11-22T23:55:23,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-22T23:55:23,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1353, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732319723511) roll requested 2024-11-22T23:55:23,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732319723518 2024-11-22T23:55:23,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732319723518 2024-11-22T23:55:23,528 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:55:23,530 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e1f1650362b6e324445573467dc3bec, disabling compactions & flushes 2024-11-22T23:55:23,530 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. 2024-11-22T23:55:23,531 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. 2024-11-22T23:55:23,531 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. after waiting 0 ms 2024-11-22T23:55:23,531 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. 2024-11-22T23:55:23,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-22T23:55:23,537 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. 2024-11-22T23:55:23,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1354, max=55; forcing (partial) flush of 1 region(s): 4e1f1650362b6e324445573467dc3bec[MyCF] 2024-11-22T23:55:23,537 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732319693170.4e1f1650362b6e324445573467dc3bec. 2024-11-22T23:55:23,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of 4e1f1650362b6e324445573467dc3bec, because it is not online on us 2024-11-22T23:55:23,537 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e1f1650362b6e324445573467dc3bec: Waiting for close lock at 1732319723530Disabling compacts and flushes for region at 1732319723530Disabling writes for close at 1732319723531 (+1 ms)Writing region close event to WAL at 1732319723536 (+5 ms)Closed at 1732319723537 (+1 ms) 2024-11-22T23:55:23,537 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:55:23,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-22T23:55:23,921 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732319723518) 2024-11-22T23:55:23,922 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9b065fe6-bf13-cb14-7051-60f80f031e2c 2024-11-22T23:55:24,586 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=325 (was 230) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8658 (was 9366)